--- /dev/null
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+*.out
+
+# Debug files
+*.dSYM/
+
+/Debug/siridb-server
+/Release/siridb-server
+*.d
+*.d
+/testdir
+*.log
+
+# Eclipse
+/.settings
+.cproject
+.project
+
+# Code:blocks
+bin/
+siridbc.cbp
+siridbc.depend
+siridbc.layout
+
+# Python
+*/__pycache__/*
+*.pyc
+
+# Build
+build/
+
+# Grammar
+grammar/cgrammar/
+grammar/jsgrammar/
+grammar/pygrammar/
+grammar/gogrammar/
+grammar/javagrammar/
+
+# Sublime Text
+*.sublime-project
+*.sublime-workspace
+
+# VS Code
+.vscode/
+
+# Test
+itest/testdir
+test/testdir
+
+# Debian
+debian/.debhelper/
+debian/debhelper-build-stamp
+debian/files
+debian/siridb-server.postrm.debhelper
+debian/siridb-server.substvars
+debian/siridb-server/
+Release/siridb-server.1
+
--- /dev/null
+language: c
+sudo: required
+dist: xenial
+before_script:
+ - sudo add-apt-repository main
+ - sudo add-apt-repository universe
+ - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+ - sudo apt-get update -qq
+ - sudo apt-get install -qq libpcre2-dev libuv1-dev libyajl-dev uuid-dev g++-7 valgrind
+ - git clone https://github.com/transceptor-technology/libcleri.git
+ - cd ./libcleri/Release/
+ - make
+ - sudo make install
+ - cd ../../Release/
+script:
+ - make test
+ - CFLAGS="-Werror -std=gnu89" make
\ No newline at end of file
--- /dev/null
+../help/
\ No newline at end of file
--- /dev/null
+-include ../makefile.init
+
+RM := rm -rf
+
+# All of the sources participating in the build are defined here
+-include sources.mk
+-include src/base64/subdir.mk
+-include src/xpath/subdir.mk
+-include src/xmath/subdir.mk
+-include src/timeit/subdir.mk
+-include src/xstr/subdir.mk
+-include src/vec/subdir.mk
+-include src/siri/service/subdir.mk
+-include src/siri/parser/subdir.mk
+-include src/siri/net/subdir.mk
+-include src/siri/help/subdir.mk
+-include src/siri/grammar/subdir.mk
+-include src/siri/file/subdir.mk
+-include src/siri/db/subdir.mk
+-include src/siri/cfg/subdir.mk
+-include src/siri/args/subdir.mk
+-include src/siri/subdir.mk
+-include src/qpack/subdir.mk
+-include src/qpjson/subdir.mk
+-include src/procinfo/subdir.mk
+-include src/owcrypt/subdir.mk
+-include src/motd/subdir.mk
+-include src/logger/subdir.mk
+-include src/lock/subdir.mk
+-include src/llist/subdir.mk
+-include src/iso8601/subdir.mk
+-include src/lib/subdir.mk
+-include src/imap/subdir.mk
+-include src/omap/subdir.mk
+-include src/expr/subdir.mk
+-include src/ctree/subdir.mk
+-include src/cfgparser/subdir.mk
+-include src/cexpr/subdir.mk
+-include src/argparse/subdir.mk
+-include subdir.mk
+-include objects.mk
+
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(strip $(C_DEPS)),)
+-include $(C_DEPS)
+endif
+endif
+
+-include ../makefile.defs
+
+OS := $(shell uname)
+ifeq ($(OS),Darwin)
+CRYPT :=
+UUID :=
+INSTALL_PATH := /usr/local
+else
+CRYPT := -lcrypt
+UUID := -luuid
+INSTALL_PATH := /usr
+endif
+
+# Add inputs and outputs from these tool invocations to the build variables
+
+# All Target
+all: siridb-server
+
+# Tool invocations
+siridb-server: $(OBJS) $(USER_OBJS)
+ @echo 'Building target: $@'
+ @echo 'Invoking: GCC C Linker'
+ gcc -o "siridb-server" $(OBJS) $(USER_OBJS) $(LIBS) $(CRYPT) $(UUID) $(LDFLAGS)
+ @echo 'Finished building target: $@'
+ @echo ' '
+
+# Other Targets
+clean:
+ -$(RM) $(EXECUTABLES)$(OBJS)$(C_DEPS) siridb-server
+ -@echo ' '
+
+.PHONY: all clean dependents
+.SECONDARY:
+
+-include ../makefile.targets
+
+test:
+ @cd ../test && ./test.sh
--- /dev/null
+USER_OBJS :=
+
+LIBS := -luv -lm -lpcre2-8 -lcleri -lyajl
+
--- /dev/null
+OBJ_SRCS :=
+ASM_SRCS :=
+C_SRCS :=
+O_SRCS :=
+S_UPPER_SRCS :=
+EXECUTABLES :=
+OBJS :=
+C_DEPS :=
+
+# Every subdirectory with source files must be described here
+SUBDIRS := \
+. \
+src/argparse \
+src/base64 \
+src/cexpr \
+src/cfgparser \
+src/ctree \
+src/expr \
+src/imap \
+src/iso8601 \
+src/lib \
+src/llist \
+src/lock \
+src/logger \
+src/omap \
+src/owcrypt \
+src/procinfo \
+src/qpack \
+src/qpjson \
+src/siri/service \
+src/siri/args \
+src/siri \
+src/siri/cfg \
+src/siri/db \
+src/siri/file \
+src/siri/grammar \
+src/siri/help \
+src/siri/net \
+src/vec \
+src/xstr \
+src/timeit \
+src/xmath \
+src/xpath \
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/argparse/argparse.c
+
+OBJS += \
+./src/argparse/argparse.o
+
+C_DEPS += \
+./src/argparse/argparse.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/argparse/%.o: ../src/argparse/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/base64/base64.c
+
+OBJS += \
+./src/base64/base64.o
+
+C_DEPS += \
+./src/base64/base64.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/base64/%.o: ../src/base64/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/cexpr/cexpr.c
+
+OBJS += \
+./src/cexpr/cexpr.o
+
+C_DEPS += \
+./src/cexpr/cexpr.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/cexpr/%.o: ../src/cexpr/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/cfgparser/cfgparser.c
+
+OBJS += \
+./src/cfgparser/cfgparser.o
+
+C_DEPS += \
+./src/cfgparser/cfgparser.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/cfgparser/%.o: ../src/cfgparser/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/ctree/ctree.c
+
+OBJS += \
+./src/ctree/ctree.o
+
+C_DEPS += \
+./src/ctree/ctree.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/ctree/%.o: ../src/ctree/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/expr/expr.c
+
+OBJS += \
+./src/expr/expr.o
+
+C_DEPS += \
+./src/expr/expr.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/expr/%.o: ../src/expr/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/imap/imap.c
+
+OBJS += \
+./src/imap/imap.o
+
+C_DEPS += \
+./src/imap/imap.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/imap/%.o: ../src/imap/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/iso8601/iso8601.c
+
+OBJS += \
+./src/iso8601/iso8601.o
+
+C_DEPS += \
+./src/iso8601/iso8601.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/iso8601/%.o: ../src/iso8601/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/lib/http_parser.c
+
+OBJS += \
+./src/lib/http_parser.o
+
+C_DEPS += \
+./src/lib/http_parser.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/lib/%.o: ../src/lib/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/llist/llist.c
+
+OBJS += \
+./src/llist/llist.o
+
+C_DEPS += \
+./src/llist/llist.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/llist/%.o: ../src/llist/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/lock/lock.c
+
+OBJS += \
+./src/lock/lock.o
+
+C_DEPS += \
+./src/lock/lock.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/lock/%.o: ../src/lock/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/logger/logger.c
+
+OBJS += \
+./src/logger/logger.o
+
+C_DEPS += \
+./src/logger/logger.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/logger/%.o: ../src/logger/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/omap/omap.c
+
+OBJS += \
+./src/omap/omap.o
+
+C_DEPS += \
+./src/omap/omap.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/omap/%.o: ../src/omap/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/owcrypt/owcrypt.c
+
+OBJS += \
+./src/owcrypt/owcrypt.o
+
+C_DEPS += \
+./src/owcrypt/owcrypt.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/owcrypt/%.o: ../src/owcrypt/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/procinfo/procinfo.c
+
+OBJS += \
+./src/procinfo/procinfo.o
+
+C_DEPS += \
+./src/procinfo/procinfo.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/procinfo/%.o: ../src/procinfo/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/qpack/qpack.c
+
+OBJS += \
+./src/qpack/qpack.o
+
+C_DEPS += \
+./src/qpack/qpack.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/qpack/%.o: ../src/qpack/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/qpjson/qpjson.c
+
+OBJS += \
+./src/qpjson/qpjson.o
+
+C_DEPS += \
+./src/qpjson/qpjson.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/qpjson/%.o: ../src/qpjson/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/args/args.c
+
+OBJS += \
+./src/siri/args/args.o
+
+C_DEPS += \
+./src/siri/args/args.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/args/%.o: ../src/siri/args/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/cfg/cfg.c
+
+OBJS += \
+./src/siri/cfg/cfg.o
+
+C_DEPS += \
+./src/siri/cfg/cfg.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/cfg/%.o: ../src/siri/cfg/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/db/access.c \
+../src/siri/db/aggregate.c \
+../src/siri/db/auth.c \
+../src/siri/db/buffer.c \
+../src/siri/db/db.c \
+../src/siri/db/ffile.c \
+../src/siri/db/fifo.c \
+../src/siri/db/forward.c \
+../src/siri/db/group.c \
+../src/siri/db/groups.c \
+../src/siri/db/initsync.c \
+../src/siri/db/insert.c \
+../src/siri/db/listener.c \
+../src/siri/db/lookup.c \
+../src/siri/db/median.c \
+../src/siri/db/misc.c \
+../src/siri/db/nodes.c \
+../src/siri/db/pcache.c \
+../src/siri/db/points.c \
+../src/siri/db/pool.c \
+../src/siri/db/pools.c \
+../src/siri/db/presuf.c \
+../src/siri/db/props.c \
+../src/siri/db/queries.c \
+../src/siri/db/query.c \
+../src/siri/db/re.c \
+../src/siri/db/reindex.c \
+../src/siri/db/replicate.c \
+../src/siri/db/series.c \
+../src/siri/db/server.c \
+../src/siri/db/servers.c \
+../src/siri/db/shard.c \
+../src/siri/db/shards.c \
+../src/siri/db/sset.c \
+../src/siri/db/tag.c \
+../src/siri/db/tags.c \
+../src/siri/db/tasks.c \
+../src/siri/db/tee.c \
+../src/siri/db/time.c \
+../src/siri/db/user.c \
+../src/siri/db/users.c \
+../src/siri/db/variance.c \
+../src/siri/db/walker.c
+
+OBJS += \
+./src/siri/db/access.o \
+./src/siri/db/aggregate.o \
+./src/siri/db/auth.o \
+./src/siri/db/buffer.o \
+./src/siri/db/db.o \
+./src/siri/db/ffile.o \
+./src/siri/db/fifo.o \
+./src/siri/db/forward.o \
+./src/siri/db/group.o \
+./src/siri/db/groups.o \
+./src/siri/db/initsync.o \
+./src/siri/db/insert.o \
+./src/siri/db/listener.o \
+./src/siri/db/lookup.o \
+./src/siri/db/median.o \
+./src/siri/db/misc.o \
+./src/siri/db/nodes.o \
+./src/siri/db/pcache.o \
+./src/siri/db/points.o \
+./src/siri/db/pool.o \
+./src/siri/db/pools.o \
+./src/siri/db/presuf.o \
+./src/siri/db/props.o \
+./src/siri/db/queries.o \
+./src/siri/db/query.o \
+./src/siri/db/re.o \
+./src/siri/db/reindex.o \
+./src/siri/db/replicate.o \
+./src/siri/db/series.o \
+./src/siri/db/server.o \
+./src/siri/db/servers.o \
+./src/siri/db/shard.o \
+./src/siri/db/shards.o \
+./src/siri/db/sset.o \
+./src/siri/db/tag.o \
+./src/siri/db/tags.o \
+./src/siri/db/tasks.o \
+./src/siri/db/tee.o \
+./src/siri/db/time.o \
+./src/siri/db/user.o \
+./src/siri/db/users.o \
+./src/siri/db/variance.o \
+./src/siri/db/walker.o
+
+C_DEPS += \
+./src/siri/db/access.d \
+./src/siri/db/aggregate.d \
+./src/siri/db/auth.d \
+./src/siri/db/buffer.d \
+./src/siri/db/db.d \
+./src/siri/db/ffile.d \
+./src/siri/db/fifo.d \
+./src/siri/db/forward.d \
+./src/siri/db/group.d \
+./src/siri/db/groups.d \
+./src/siri/db/initsync.d \
+./src/siri/db/insert.d \
+./src/siri/db/listener.d \
+./src/siri/db/lookup.d \
+./src/siri/db/median.d \
+./src/siri/db/misc.d \
+./src/siri/db/nodes.d \
+./src/siri/db/pcache.d \
+./src/siri/db/points.d \
+./src/siri/db/pool.d \
+./src/siri/db/pools.d \
+./src/siri/db/presuf.d \
+./src/siri/db/props.d \
+./src/siri/db/queries.d \
+./src/siri/db/query.d \
+./src/siri/db/re.d \
+./src/siri/db/reindex.d \
+./src/siri/db/replicate.d \
+./src/siri/db/series.d \
+./src/siri/db/server.d \
+./src/siri/db/servers.d \
+./src/siri/db/shard.d \
+./src/siri/db/shards.d \
+./src/siri/db/sset.d \
+./src/siri/db/tag.d \
+./src/siri/db/tags.d \
+./src/siri/db/tasks.d \
+./src/siri/db/tee.d \
+./src/siri/db/time.d \
+./src/siri/db/user.d \
+./src/siri/db/users.d \
+./src/siri/db/variance.d \
+./src/siri/db/walker.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/db/%.o: ../src/siri/db/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/file/handler.c \
+../src/siri/file/pointer.c
+
+OBJS += \
+./src/siri/file/handler.o \
+./src/siri/file/pointer.o
+
+C_DEPS += \
+./src/siri/file/handler.d \
+./src/siri/file/pointer.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/file/%.o: ../src/siri/file/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/grammar/grammar.c
+
+OBJS += \
+./src/siri/grammar/grammar.o
+
+C_DEPS += \
+./src/siri/grammar/grammar.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/grammar/%.o: ../src/siri/grammar/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/help/help.c
+
+OBJS += \
+./src/siri/help/help.o
+
+C_DEPS += \
+./src/siri/help/help.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/help/%.o: ../src/siri/help/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/net/bserver.c \
+../src/siri/net/clserver.c \
+../src/siri/net/pkg.c \
+../src/siri/net/promise.c \
+../src/siri/net/promises.c \
+../src/siri/net/protocol.c \
+../src/siri/net/stream.c \
+../src/siri/net/tcp.c \
+../src/siri/net/pipe.c
+
+OBJS += \
+./src/siri/net/bserver.o \
+./src/siri/net/clserver.o \
+./src/siri/net/pkg.o \
+./src/siri/net/promise.o \
+./src/siri/net/promises.o \
+./src/siri/net/protocol.o \
+./src/siri/net/stream.o \
+./src/siri/net/tcp.o \
+./src/siri/net/pipe.o
+
+C_DEPS += \
+./src/siri/net/bserver.d \
+./src/siri/net/clserver.d \
+./src/siri/net/pkg.d \
+./src/siri/net/promise.d \
+./src/siri/net/promises.d \
+./src/siri/net/protocol.d \
+./src/siri/net/stream.d \
+./src/siri/net/tcp.d \
+./src/siri/net/pipe.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/net/%.o: ../src/siri/net/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/service/account.c \
+../src/siri/service/client.c \
+../src/siri/service/request.c
+
+OBJS += \
+./src/siri/service/account.o \
+./src/siri/service/client.o \
+./src/siri/service/request.o
+
+C_DEPS += \
+./src/siri/service/account.d \
+./src/siri/service/client.d \
+./src/siri/service/request.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/service/%.o: ../src/siri/service/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/api.c \
+../src/siri/async.c \
+../src/siri/backup.c \
+../src/siri/buffersync.c \
+../src/siri/err.c \
+../src/siri/evars.c \
+../src/siri/health.c \
+../src/siri/heartbeat.c \
+../src/siri/optimize.c \
+../src/siri/siri.c \
+../src/siri/version.c
+
+OBJS += \
+./src/siri/api.o \
+./src/siri/async.o \
+./src/siri/backup.o \
+./src/siri/buffersync.o \
+./src/siri/err.o \
+./src/siri/evars.o \
+./src/siri/health.o \
+./src/siri/heartbeat.o \
+./src/siri/optimize.o \
+./src/siri/siri.o \
+./src/siri/version.o
+
+C_DEPS += \
+./src/siri/api.d \
+./src/siri/async.d \
+./src/siri/backup.d \
+./src/siri/buffersync.d \
+./src/siri/err.d \
+./src/siri/evars.d \
+./src/siri/health.d \
+./src/siri/heartbeat.d \
+./src/siri/optimize.d \
+./src/siri/siri.d \
+./src/siri/version.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/%.o: ../src/siri/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/timeit/timeit.c
+
+OBJS += \
+./src/timeit/timeit.o
+
+C_DEPS += \
+./src/timeit/timeit.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/timeit/%.o: ../src/timeit/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/vec/vec.c
+
+OBJS += \
+./src/vec/vec.o
+
+C_DEPS += \
+./src/vec/vec.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/vec/%.o: ../src/vec/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/xmath/xmath.c
+
+OBJS += \
+./src/xmath/xmath.o
+
+C_DEPS += \
+./src/xmath/xmath.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/xmath/%.o: ../src/xmath/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/xpath/xpath.c
+
+OBJS += \
+./src/xpath/xpath.o
+
+C_DEPS += \
+./src/xpath/xpath.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/xpath/%.o: ../src/xpath/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/xstr/xstr.c
+
+OBJS += \
+./src/xstr/xstr.o
+
+C_DEPS += \
+./src/xstr/xstr.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/xstr/%.o: ../src/xstr/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../main.c
+
+OBJS += \
+./main.o
+
+C_DEPS += \
+./main.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+%.o: ../%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -I../include -O0 -g3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+MIT License
+
+Copyright (c) 2016 Transceptor Technology
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- /dev/null
+SiriDB Server
+=============
+SiriDB is a highly-scalable, robust and super fast time series database.
+
+---------------------------------------
+ * [Installation](#installation)
+ * [Ubuntu](#ubuntu)
+ * [Compile from source](#compile-from-source)
+ * [Linux](#linux)
+ * [OSX](#osx)
+ * [Configuration](#configuration)
+ * [Build Debian package](#build-debian-package)
+ * [Run integration tests](#run-integration-tests)
+ * [Create or expand a database](#create-or-expand-a-database)
+ * [Using SiriDB](#using-siridb)
+ * [SiriDB Connectors](#siridb-connectors)
+ * [SiriDB HTTP](#siridb-http)
+ * [SiriDB Prompt](#siridb-prompt)
+ * [Grafana](#grafana)
+ * [API/Query language](#query-language)
+
+---------------------------------------
+
+## Installation
+### Ubuntu
+For Ubuntu we have a deb package available which can be downloaded [here](https://github.com/SiriDB/siridb-server/releases/latest).
+
+Note: SiriDB requires *libexpat1*, *libuv1*, *libpcre2-8-0* and *libcleri0* these libraries can be easily installed using apt:
+```
+apt install libexpat1 libuv1 libpcre2-8-0 libcleri0
+```
+
+>Library `libcleri0` is available from Ubuntu 18.04, for older versions a deb package can be found here:
+>https://github.com/transceptor-technology/libcleri/releases/latest
+
+The .deb package installs a configuration file at `/etc/siridb/siridb.conf`. You might want to view or change this file before starting SiriDB.
+
+### Compile from source
+>From version 2.0.19 libcleri is not included as part of this source anymore
+>and needs to be installed separately. libcleri can be found here:
+>[https://github.com/transceptor-technology/libcleri](https://github.com/transceptor-technology/libcleri)
+>or can be installed using `apt`.
+
+#### Linux
+Install the following requirements: (Ubuntu 18.04)
+```
+sudo apt install libcleri-dev
+sudo apt install libpcre2-dev
+sudo apt install libuv1-dev
+sudo apt install libyajl-dev
+sudo apt install uuid-dev
+```
+
+Compile (replace Release with Debug for a debug build):
+```
+cd ./Release
+make clean
+make test
+make
+```
+
+Install
+```
+sudo make install
+```
+
+#### OSX
+>Make sure [libcleri](https://github.com/transceptor-technology/libcleri) is installed!
+
+Install the following requirements:
+```
+brew install pcre2
+brew install libuv
+brew install yajl
+brew install ossp-uuid
+```
+Compile (replace Release with Debug for a debug build):
+```
+cd ./Release
+export CFLAGS="-I/usr/local/include"
+export LDFLAGS="-L/usr/local/lib"
+make clean
+make test
+make
+```
+
+Install
+```
+sudo make install
+```
+
+#### Configuration
+SiriDB requires a configuration file to run. By default SiriDB will search for the configuration file in `/etc/siridb/siridb.conf` but alternatively you can specify a custom path by using the `-c/--config` argument.
+
+```
+$ siridb-server -c /my/path/siridb.conf
+```
+An example configuration file can be found here:
+[https://github.com/SiriDB/siridb-server/blob/master/siridb.conf](https://github.com/SiriDB/siridb-server/blob/master/siridb.conf)
+
+### Build Debian package:
+
+Install required packages (*autopkgtest is required for running the tests*)
+```
+apt-get install devscripts lintian help2man autopkgtest
+```
+
+Create archive
+```
+git archive -o ../siridb-server_2.0.31.orig.tar.gz master
+```
+
+Run tests
+```
+autopkgtest -B -- null
+```
+
+Build package
+```
+debuild -us -uc
+```
+
+## Run integration tests
+The simplest way to run the integration tests is to use [docker](https://docs.docker.com/install/).
+
+Build integration test image
+```
+docker build -t siridb/itest -f itest/Dockerfile .
+```
+
+Run integration tests
+```
+docker run siridb/itest:latest
+```
+
+## Create or expand a database
+[SiriDB Admin](https://github.com/SiriDB/siridb-admin) can be used for creating a new database or expanding an existing database with a new server. Documentation on how to install and use the admin tool can be found at the [siridb-admin](https://github.com/SiriDB/siridb-admin#readme) github project. Binaries are available for most platforms and can be downloaded from [here](https://github.com/SiriDB/siridb-admin/releases/latest). As an alternative it is possible to use a simple [HTTP API](https://docs.siridb.net/connect/http_api/) for creating or expanding a SiriDB database.
+
+## Using SiriDB
+SiriDB has several tools available to connect to a SiriDB database.
+
+### SiriDB Connectors
+The following native connectors are available:
+ - [C/C++](https://github.com/SiriDB/libsiridb#readme)
+ - [Python](https://github.com/SiriDB/siridb-connector#readme)
+ - [Go](https://github.com/SiriDB/go-siridb-connector#readme)
+ - [Node.js](https://github.com/SiriDB/siridb-nodejs-addon#readme)
+
+When not using one of the above, you can still communicate to SiriDB using [SiriDB HTTP](#siridb-http).
+
+### SiriDB HTTP
+[SiriDB HTTP](https://github.com/SiriDB/siridb-http#readme) provides a HTTP API for SiriDB and has support for JSON, MsgPack, Qpack, CSV and Socket.io. SiriDB HTTP also has an optional web interface and SSL support.
+
+### SiriDB Prompt
+[SiriDB Prompt](https://github.com/SiriDB/siridb-prompt#readme) can be used as a command line SiriDB client with auto-completion support and can be used to load json or csv data into a SiriDB database. Click [here](https://github.com/SiriDB/siridb-prompt/blob/master/README.md) for more information about SiriDB Prompt.
+
+### Grafana
+[SiriDB Grafana Datasource](https://github.com/SiriDB/grafana-siridb-http-datasource#readme) is a plugin for Grafana. See the following blog article on how to configure and use this plugin: https://github.com/SiriDB/grafana-siridb-http-example.
+
+## Query language
+Documentation about the query language can be found at https://siridb.net/documentation.
--- /dev/null
+../help/
\ No newline at end of file
--- /dev/null
+-include ../makefile.init
+
+RM := rm -rf
+
+# All of the sources participating in the build are defined here
+-include sources.mk
+-include src/base64/subdir.mk
+-include src/xpath/subdir.mk
+-include src/xmath/subdir.mk
+-include src/timeit/subdir.mk
+-include src/xstr/subdir.mk
+-include src/vec/subdir.mk
+-include src/siri/service/subdir.mk
+-include src/siri/parser/subdir.mk
+-include src/siri/net/subdir.mk
+-include src/siri/help/subdir.mk
+-include src/siri/grammar/subdir.mk
+-include src/siri/file/subdir.mk
+-include src/siri/db/subdir.mk
+-include src/siri/cfg/subdir.mk
+-include src/siri/args/subdir.mk
+-include src/siri/subdir.mk
+-include src/qpack/subdir.mk
+-include src/qpjson/subdir.mk
+-include src/procinfo/subdir.mk
+-include src/owcrypt/subdir.mk
+-include src/motd/subdir.mk
+-include src/logger/subdir.mk
+-include src/lock/subdir.mk
+-include src/llist/subdir.mk
+-include src/iso8601/subdir.mk
+-include src/lib/subdir.mk
+-include src/imap/subdir.mk
+-include src/omap/subdir.mk
+-include src/expr/subdir.mk
+-include src/ctree/subdir.mk
+-include src/cfgparser/subdir.mk
+-include src/cexpr/subdir.mk
+-include src/argparse/subdir.mk
+-include subdir.mk
+-include objects.mk
+
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(strip $(C_DEPS)),)
+-include $(C_DEPS)
+endif
+endif
+
+-include ../makefile.defs
+
+OS := $(shell uname)
+ifeq ($(OS),Darwin)
+CRYPT :=
+UUID :=
+INSTALL_PATH := /usr/local
+else
+CRYPT := -lcrypt
+UUID := -luuid
+INSTALL_PATH := /usr
+endif
+
+# Add inputs and outputs from these tool invocations to the build variables
+
+# All Target
+all: siridb-server
+
+# Tool invocations
+siridb-server: $(OBJS) $(USER_OBJS)
+ @echo 'Building target: $@'
+ @echo 'Invoking: GCC C Linker'
+ gcc -o "siridb-server" $(OBJS) $(USER_OBJS) $(LDFLAGS) $(LIBS) $(CRYPT) $(UUID)
+ @echo 'Finished building target: $@'
+ @echo ' '
+
+# Other Targets
+clean:
+ -$(RM) $(EXECUTABLES)$(OBJS)$(C_DEPS) siridb-server
+ -@echo ' '
+
+.PHONY: all clean dependents
+.SECONDARY:
+
+-include ../makefile.targets
+
+test:
+ @cd ../test && ./test.sh
--- /dev/null
+USER_OBJS :=
+
+LIBS := -luv -lm -lpcre2-8 -lcleri -lyajl
+
--- /dev/null
+OBJ_SRCS :=
+ASM_SRCS :=
+C_SRCS :=
+O_SRCS :=
+S_UPPER_SRCS :=
+EXECUTABLES :=
+OBJS :=
+C_DEPS :=
+
+# Every subdirectory with source files must be described here
+SUBDIRS := \
+. \
+src/argparse \
+src/base64 \
+src/cexpr \
+src/cfgparser \
+src/ctree \
+src/expr \
+src/imap \
+src/iso8601 \
+src/lib \
+src/llist \
+src/lock \
+src/logger \
+src/omap \
+src/owcrypt \
+src/procinfo \
+src/qpack \
+src/qpjson \
+src/siri/service \
+src/siri/args \
+src/siri \
+src/siri/cfg \
+src/siri/db \
+src/siri/file \
+src/siri/grammar \
+src/siri/help \
+src/siri/net \
+src/vec \
+src/xstr \
+src/timeit \
+src/xmath \
+src/xpath \
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/argparse/argparse.c
+
+OBJS += \
+./src/argparse/argparse.o
+
+C_DEPS += \
+./src/argparse/argparse.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/argparse/%.o: ../src/argparse/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/base64/base64.c
+
+OBJS += \
+./src/base64/base64.o
+
+C_DEPS += \
+./src/base64/base64.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/base64/%.o: ../src/base64/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/cexpr/cexpr.c
+
+OBJS += \
+./src/cexpr/cexpr.o
+
+C_DEPS += \
+./src/cexpr/cexpr.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/cexpr/%.o: ../src/cexpr/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/cfgparser/cfgparser.c
+
+OBJS += \
+./src/cfgparser/cfgparser.o
+
+C_DEPS += \
+./src/cfgparser/cfgparser.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/cfgparser/%.o: ../src/cfgparser/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/ctree/ctree.c
+
+OBJS += \
+./src/ctree/ctree.o
+
+C_DEPS += \
+./src/ctree/ctree.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/ctree/%.o: ../src/ctree/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/expr/expr.c
+
+OBJS += \
+./src/expr/expr.o
+
+C_DEPS += \
+./src/expr/expr.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/expr/%.o: ../src/expr/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/imap/imap.c
+
+OBJS += \
+./src/imap/imap.o
+
+C_DEPS += \
+./src/imap/imap.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/imap/%.o: ../src/imap/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/iso8601/iso8601.c
+
+OBJS += \
+./src/iso8601/iso8601.o
+
+C_DEPS += \
+./src/iso8601/iso8601.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/iso8601/%.o: ../src/iso8601/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/lib/http_parser.c
+
+OBJS += \
+./src/lib/http_parser.o
+
+C_DEPS += \
+./src/lib/http_parser.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/lib/%.o: ../src/lib/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/llist/llist.c
+
+OBJS += \
+./src/llist/llist.o
+
+C_DEPS += \
+./src/llist/llist.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/llist/%.o: ../src/llist/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/lock/lock.c
+
+OBJS += \
+./src/lock/lock.o
+
+C_DEPS += \
+./src/lock/lock.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/lock/%.o: ../src/lock/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/logger/logger.c
+
+OBJS += \
+./src/logger/logger.o
+
+C_DEPS += \
+./src/logger/logger.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/logger/%.o: ../src/logger/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/omap/omap.c
+
+OBJS += \
+./src/omap/omap.o
+
+C_DEPS += \
+./src/omap/omap.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/omap/%.o: ../src/omap/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/owcrypt/owcrypt.c
+
+OBJS += \
+./src/owcrypt/owcrypt.o
+
+C_DEPS += \
+./src/owcrypt/owcrypt.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/owcrypt/%.o: ../src/owcrypt/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/procinfo/procinfo.c
+
+OBJS += \
+./src/procinfo/procinfo.o
+
+C_DEPS += \
+./src/procinfo/procinfo.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/procinfo/%.o: ../src/procinfo/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/qpack/qpack.c
+
+OBJS += \
+./src/qpack/qpack.o
+
+C_DEPS += \
+./src/qpack/qpack.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/qpack/%.o: ../src/qpack/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/qpjson/qpjson.c
+
+OBJS += \
+./src/qpjson/qpjson.o
+
+C_DEPS += \
+./src/qpjson/qpjson.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/qpjson/%.o: ../src/qpjson/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/args/args.c
+
+OBJS += \
+./src/siri/args/args.o
+
+C_DEPS += \
+./src/siri/args/args.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/args/%.o: ../src/siri/args/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/cfg/cfg.c
+
+OBJS += \
+./src/siri/cfg/cfg.o
+
+C_DEPS += \
+./src/siri/cfg/cfg.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/cfg/%.o: ../src/siri/cfg/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/db/access.c \
+../src/siri/db/aggregate.c \
+../src/siri/db/auth.c \
+../src/siri/db/buffer.c \
+../src/siri/db/db.c \
+../src/siri/db/ffile.c \
+../src/siri/db/fifo.c \
+../src/siri/db/forward.c \
+../src/siri/db/group.c \
+../src/siri/db/groups.c \
+../src/siri/db/initsync.c \
+../src/siri/db/insert.c \
+../src/siri/db/listener.c \
+../src/siri/db/lookup.c \
+../src/siri/db/median.c \
+../src/siri/db/misc.c \
+../src/siri/db/nodes.c \
+../src/siri/db/pcache.c \
+../src/siri/db/points.c \
+../src/siri/db/pool.c \
+../src/siri/db/pools.c \
+../src/siri/db/presuf.c \
+../src/siri/db/props.c \
+../src/siri/db/queries.c \
+../src/siri/db/query.c \
+../src/siri/db/re.c \
+../src/siri/db/reindex.c \
+../src/siri/db/replicate.c \
+../src/siri/db/series.c \
+../src/siri/db/server.c \
+../src/siri/db/servers.c \
+../src/siri/db/shard.c \
+../src/siri/db/shards.c \
+../src/siri/db/sset.c \
+../src/siri/db/tag.c \
+../src/siri/db/tags.c \
+../src/siri/db/tasks.c \
+../src/siri/db/tee.c \
+../src/siri/db/time.c \
+../src/siri/db/user.c \
+../src/siri/db/users.c \
+../src/siri/db/variance.c \
+../src/siri/db/walker.c
+
+OBJS += \
+./src/siri/db/access.o \
+./src/siri/db/aggregate.o \
+./src/siri/db/auth.o \
+./src/siri/db/buffer.o \
+./src/siri/db/db.o \
+./src/siri/db/ffile.o \
+./src/siri/db/fifo.o \
+./src/siri/db/forward.o \
+./src/siri/db/group.o \
+./src/siri/db/groups.o \
+./src/siri/db/initsync.o \
+./src/siri/db/insert.o \
+./src/siri/db/listener.o \
+./src/siri/db/lookup.o \
+./src/siri/db/median.o \
+./src/siri/db/misc.o \
+./src/siri/db/nodes.o \
+./src/siri/db/pcache.o \
+./src/siri/db/points.o \
+./src/siri/db/pool.o \
+./src/siri/db/pools.o \
+./src/siri/db/presuf.o \
+./src/siri/db/props.o \
+./src/siri/db/queries.o \
+./src/siri/db/query.o \
+./src/siri/db/re.o \
+./src/siri/db/reindex.o \
+./src/siri/db/replicate.o \
+./src/siri/db/series.o \
+./src/siri/db/server.o \
+./src/siri/db/servers.o \
+./src/siri/db/shard.o \
+./src/siri/db/shards.o \
+./src/siri/db/sset.o \
+./src/siri/db/tag.o \
+./src/siri/db/tags.o \
+./src/siri/db/tasks.o \
+./src/siri/db/tee.o \
+./src/siri/db/time.o \
+./src/siri/db/user.o \
+./src/siri/db/users.o \
+./src/siri/db/variance.o \
+./src/siri/db/walker.o
+
+C_DEPS += \
+./src/siri/db/access.d \
+./src/siri/db/aggregate.d \
+./src/siri/db/auth.d \
+./src/siri/db/buffer.d \
+./src/siri/db/db.d \
+./src/siri/db/ffile.d \
+./src/siri/db/fifo.d \
+./src/siri/db/forward.d \
+./src/siri/db/group.d \
+./src/siri/db/groups.d \
+./src/siri/db/initsync.d \
+./src/siri/db/insert.d \
+./src/siri/db/listener.d \
+./src/siri/db/lookup.d \
+./src/siri/db/median.d \
+./src/siri/db/misc.d \
+./src/siri/db/nodes.d \
+./src/siri/db/pcache.d \
+./src/siri/db/points.d \
+./src/siri/db/pool.d \
+./src/siri/db/pools.d \
+./src/siri/db/presuf.d \
+./src/siri/db/props.d \
+./src/siri/db/queries.d \
+./src/siri/db/query.d \
+./src/siri/db/re.d \
+./src/siri/db/reindex.d \
+./src/siri/db/replicate.d \
+./src/siri/db/series.d \
+./src/siri/db/server.d \
+./src/siri/db/servers.d \
+./src/siri/db/shard.d \
+./src/siri/db/shards.d \
+./src/siri/db/sset.d \
+./src/siri/db/tag.d \
+./src/siri/db/tags.d \
+./src/siri/db/tasks.d \
+./src/siri/db/tee.d \
+./src/siri/db/time.d \
+./src/siri/db/user.d \
+./src/siri/db/users.d \
+./src/siri/db/variance.d \
+./src/siri/db/walker.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/db/%.o: ../src/siri/db/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/file/handler.c \
+../src/siri/file/pointer.c
+
+OBJS += \
+./src/siri/file/handler.o \
+./src/siri/file/pointer.o
+
+C_DEPS += \
+./src/siri/file/handler.d \
+./src/siri/file/pointer.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/file/%.o: ../src/siri/file/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/grammar/grammar.c
+
+OBJS += \
+./src/siri/grammar/grammar.o
+
+C_DEPS += \
+./src/siri/grammar/grammar.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/grammar/%.o: ../src/siri/grammar/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/help/help.c
+
+OBJS += \
+./src/siri/help/help.o
+
+C_DEPS += \
+./src/siri/help/help.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/help/%.o: ../src/siri/help/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/net/bserver.c \
+../src/siri/net/clserver.c \
+../src/siri/net/pkg.c \
+../src/siri/net/promise.c \
+../src/siri/net/promises.c \
+../src/siri/net/protocol.c \
+../src/siri/net/stream.c \
+../src/siri/net/tcp.c \
+../src/siri/net/pipe.c
+
+OBJS += \
+./src/siri/net/bserver.o \
+./src/siri/net/clserver.o \
+./src/siri/net/pkg.o \
+./src/siri/net/promise.o \
+./src/siri/net/promises.o \
+./src/siri/net/protocol.o \
+./src/siri/net/stream.o \
+./src/siri/net/tcp.o \
+./src/siri/net/pipe.o
+
+C_DEPS += \
+./src/siri/net/bserver.d \
+./src/siri/net/clserver.d \
+./src/siri/net/pkg.d \
+./src/siri/net/promise.d \
+./src/siri/net/promises.d \
+./src/siri/net/protocol.d \
+./src/siri/net/stream.d \
+./src/siri/net/tcp.d \
+./src/siri/net/pipe.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/net/%.o: ../src/siri/net/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/service/account.c \
+../src/siri/service/client.c \
+../src/siri/service/request.c
+
+OBJS += \
+./src/siri/service/account.o \
+./src/siri/service/client.o \
+./src/siri/service/request.o
+
+C_DEPS += \
+./src/siri/service/account.d \
+./src/siri/service/client.d \
+./src/siri/service/request.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/args/%.o: ../src/siri/args/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/siri/api.c \
+../src/siri/async.c \
+../src/siri/backup.c \
+../src/siri/buffersync.c \
+../src/siri/err.c \
+../src/siri/evars.c \
+../src/siri/health.c \
+../src/siri/heartbeat.c \
+../src/siri/optimize.c \
+../src/siri/siri.c \
+../src/siri/version.c
+
+OBJS += \
+./src/siri/api.o \
+./src/siri/async.o \
+./src/siri/backup.o \
+./src/siri/buffersync.o \
+./src/siri/err.o \
+./src/siri/evars.o \
+./src/siri/health.o \
+./src/siri/heartbeat.o \
+./src/siri/optimize.o \
+./src/siri/siri.o \
+./src/siri/version.o
+
+C_DEPS += \
+./src/siri/api.d \
+./src/siri/async.d \
+./src/siri/backup.d \
+./src/siri/buffersync.d \
+./src/siri/err.d \
+./src/siri/evars.d \
+./src/siri/health.d \
+./src/siri/heartbeat.d \
+./src/siri/optimize.d \
+./src/siri/siri.d \
+./src/siri/version.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/siri/%.o: ../src/siri/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/timeit/timeit.c
+
+OBJS += \
+./src/timeit/timeit.o
+
+C_DEPS += \
+./src/timeit/timeit.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/timeit/%.o: ../src/timeit/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/vec/vec.c
+
+OBJS += \
+./src/vec/vec.o
+
+C_DEPS += \
+./src/vec/vec.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/vec/%.o: ../src/vec/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/xmath/xmath.c
+
+OBJS += \
+./src/xmath/xmath.o
+
+C_DEPS += \
+./src/xmath/xmath.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/xmath/%.o: ../src/xmath/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/xpath/xpath.c
+
+OBJS += \
+./src/xpath/xpath.o
+
+C_DEPS += \
+./src/xpath/xpath.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/xpath/%.o: ../src/xpath/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../src/xstr/xstr.c
+
+OBJS += \
+./src/xstr/xstr.o
+
+C_DEPS += \
+./src/xstr/xstr.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+src/xstr/%.o: ../src/xstr/%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+# Add inputs and outputs from these tool invocations to the build variables
+C_SRCS += \
+../main.c
+
+OBJS += \
+./main.o
+
+C_DEPS += \
+./main.d
+
+
+# Each subdirectory must supply rules for building sources it contributes
+%.o: ../%.c
+ @echo 'Building file: $<'
+ @echo 'Invoking: GCC C Compiler'
+ gcc -DNDEBUG -I../include -O3 -Wall -Wextra $(CPPFLAGS) $(CFLAGS) -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@)" -o "$@" "$<"
+ @echo 'Finished building: $<'
+ @echo ' '
+
+
--- /dev/null
+siridb-server (2.0.43-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Fixed HTTP pipeline issue (@srdgame, #161)
+ - Fixed possible memory corruption (@ubnt-michals, #158)
+ - Changed SIRIDB_DB_PATH configuration (#154)
+ - Fixed shard duration for ARM builds (@srdgame, #164)
+ - Use IPv4 for health and API when configured (@ubnt-michals, #155)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Thu, 04 Feb 2021 15:36:02 +0100
+
+siridb-server (2.0.42-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Fixed handle ready status in Kubernetes statefulset (#153)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Thu, 12 Nov 2020 09:39:13 +0100
+
+siridb-server (2.0.41-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Fixed bug in insert loop (#147)
+ - Fixed loading databases with duplicated series (#148)
+ - Allow server names without defining a port (#150)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Wed, 04 Nov 2020 13:15:29 +0100
+
+siridb-server (2.0.40-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Added `alter tag` syntax (#144)
+ - Fixed list tags error when conditions are used (#145)
+ - Fixed division by zero bug (#146)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Fri, 25 Sep 2020 16:35:28 +0200
+
+siridb-server (2.0.39-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Added auto shard duration option (#141)
+ - Added `shard_duration` property on list series (#140)
+ - Added `timeval()` and `interval()` functions (#138)
+ - Fixed build error on 32 bit systems (#135)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Fri, 18 Sep 2020 15:50:10 +0200
+
+siridb-server (2.0.38-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Added tag support (#74)
+ - Fixed cleanup duplicate package (#134)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Fri, 28 Aug 2020 11:12:24 +0200
+
+siridb-server (2.0.37-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Fixed reading `series.dat` after unclean shutdown (#130)
+ - Fixed `gcc-10` compile errors (#132)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Thu, 23 Jul 2020 14:38:50 +0200
+
+siridb-server (2.0.36-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Fixed bug in max open files setting (#125)
+ - Accept Grafana compatible API requests (#129)
+ - Added support for configuration using environment variable (#128)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Wed, 22 Apr 2020 11:18:32 +0200
+
+siridb-server (2.0.35-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Added shard expiration (#123)
+ - Added HTTP API (#124)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Mon, 24 Feb 2020 16:23:37 +0100
+
+siridb-server (2.0.34-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Added readiness and liveness HTTP handler (#122)
+ - Fixed bug in sending packages to multiple SiriDB servers (#120)
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Mon, 08 Jul 2019 14:18:56 +0200
+
+siridb-server (2.0.33-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Fixed deep recursion in series selection (#118)
+ - Fixed a few gcc version 8 warnings (#117)
+ - Increased package size limit and warning
+ - Added check for unexpected time-stamps in shards at startup
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Thu, 04 Apr 2019 14:19:22 +0200
+
+siridb-server (2.0.32-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Fixed drop database
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Fri, 11 Jan 2019 16:03:42 +0100
+
+siridb-server (2.0.31-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - Added option for adding a data tee
+ - Added option to drop a database (#115)
+ - Remove limit of 4 database (#114)
+ - Do not return with an error if series do not exist (#112)
+ - Allow the use of parentheses in selecting series (#111)
+ - Fixed bug in merging series with high time precision (#108)
+ - Code forward compatible with upcoming cleri release
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Thu, 10 Jan 2019 11:35:12 +0100
+
+siridb-server (2.0.30-0~tt1) unstable; urgency=medium
+
+ * New upstream release
+ - SiriDB Server can now compile with gnu89/gnu90 (#101)
+ - Removed deprecated info- and loaddb requests
+ - Added named pipe support (#104, @pavelxdd)
+ - Changed writing buffer length to reduce random io
+ - Added option to fsync the buffer on a configurable interval
+ - Use posix_fadvise() on the buffer file (@Svedrin)
+ - The buffer size can now be adjusted by using the database.conf
+ configuration file
+ - Added conversion of invalid value types
+
+ -- Jeroen van der Heijden <jeroen@transceptor.technology> Fri, 12 Oct 2018 19:44:24 +0200
+
+siridb-server (2.0.29-1) unstable; urgency=medium
+
+ * New upstream release
+
+ -- Paul Gevers <elbrus@debian.org> Wed, 25 Jul 2018 21:57:12 +0200
+
+siridb-server (2.0.28-1) unstable; urgency=medium
+
+ * New upstream release
+
+ -- Paul Gevers <elbrus@debian.org> Mon, 02 Jul 2018 14:00:34 +0200
+
+siridb-server (2.0.27-1) unstable; urgency=medium
+
+ * New upstream release
+ * Update Maintainer and Vcs fields (Closes: #890700)
+
+ -- Paul Gevers <elbrus@debian.org> Mon, 28 May 2018 10:22:52 +0200
+
+siridb-server (2.0.26-1) unstable; urgency=medium
+
+ * New upstream release
+ * Drop all patches
+ * Add man page
+
+ -- Paul Gevers <elbrus@debian.org> Sat, 06 Jan 2018 07:54:21 +0100
+
+siridb-server (2.0.25-1) unstable; urgency=medium
+
+ * Initial release. (Closes: #882678)
+
+ -- Paul Gevers <elbrus@debian.org> Mon, 04 Dec 2017 21:40:02 +0100
--- /dev/null
+Release/siridb-server.1
--- /dev/null
+Source: siridb-server
+Section: database
+Priority: optional
+Maintainer: SiriDB Maintainers <team+debian-siridb-packaging-team@tracker.debian.org>
+Uploaders:
+ Jeroen van der Heijden <jeroen@transceptor.technology>,
+ Paul Gevers <elbrus@debian.org>,
+Rules-Requires-Root: no
+Build-Depends:
+ debhelper (>= 11~),
+ help2man,
+ libcleri-dev,
+ libpcre2-dev,
+ libuv1-dev,
+ uuid-dev,
+Homepage: https://siridb.net/
+Vcs-Browser: https://salsa.debian.org/siridb-team/siridb-server
+Vcs-Git: https://salsa.debian.org/siridb-team/siridb-server.git
+Standards-Version: 4.1.3
+
+Package: siridb-server
+Architecture: any
+Depends:
+ ucf,
+ ${misc:Depends},
+ ${shlibs:Depends},
+ libuv1 (>= 1.8.0),
+ libcleri0 (>= 0.9.3),
+Description: SiriDB time series database server
+ SiriDB is a scalable, robust and fast time series database. Build from the
+ ground up SiriDB uses a mechanism to operate without a global index and allows
+ server resources to be added on the fly. SiriDB's query language includes
+ dynamic grouping of time series for easy analysis over large amounts of time
+ series.
--- /dev/null
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+
+Files: *
+Copyright: 2016-2018, Transceptor Technology <info@transceptor.technology>
+License: Expat
+
+Files: debian/*
+Copyright: 2017-2018 Paul Gevers <elbrus@debian.org>
+License: Expat
+
+License: Expat
+ The MIT License
+ .
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated
+ documentation files (the "Software"), to deal in the Software
+ without restriction, including without limitation the rights to
+ use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to
+ whom the Software is furnished to do so, subject to the
+ following conditions:
+ .
+ The above copyright notice and this permission notice shall
+ be included in all copies or substantial portions of the
+ Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT
+ WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+ SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
--- /dev/null
+etc/siridb
+var/lib/siridb
--- /dev/null
+README.md
+docs/*
--- /dev/null
+Release/siridb-server usr/lib/siridb-server
+help usr/share/siridb-server
+siridb.conf usr/share/siridb-server/conf_templates
--- /dev/null
+usr/lib/siridb-server/siridb-server usr/bin/siridb-server
+usr/share/siridb-server/help usr/lib/siridb-server/help
+usr/share/siridb-server/help usr/share/doc/siridb-server/help
--- /dev/null
+Release/siridb-server.1
--- /dev/null
+#!/bin/sh
+set -e
+
+ucf --debconf-ok /usr/share/siridb-server/conf_templates/siridb.conf /etc/siridb/siridb.conf
+ucfr siridb-server /etc/siridb/siridb.conf
+
+#DEBHELPER#
+exit 0
--- /dev/null
+#!/bin/sh
+set -e
+
+config_file=/etc/siridb/siridb.conf
+
+case "$1" in
+ purge)
+ [ -d "/var/lib/siridb" ] && rm -rf /var/lib/siridb
+ if which ucf >/dev/null 2>&1; then
+ ucf --purge $config_file
+ fi
+ if [ -x "`which ucfr 2>/dev/null`" ]; then
+ ucfr --purge siridb-server $config_file
+ fi
+ for ext in .ucf-new .ucf-old .ucf-dist ""; do
+ rm -f "$config_file$ext"
+ done
+ ;;
+ remove)
+ ;;
+esac
+
+#DEBHELPER#
+exit 0
--- /dev/null
+#!/usr/bin/make -f
+
+export DEB_BUILD_MAINT_OPTIONS=hardening=+all
+
+%:
+ dh $@
+
+override_dh_auto_build-arch:
+ $(MAKE) --directory=Release
+ help2man -N -n"time series database server" Release/siridb-server > Release/siridb-server.1
+
+override_dh_auto_clean:
+ $(MAKE) --directory=Release clean
+ dh_auto_clean
--- /dev/null
+[Unit]
+Description=SiriDB Server
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/siridb-server --config /etc/siridb/siridb.conf --log-level warning
+StandardOutput=journal
+LimitNOFILE=65535
+TimeoutStartSec=10
+TimeoutStopSec=300
+
+[Install]
+WantedBy=multi-user.target
--- /dev/null
+3.0 (quilt)
--- /dev/null
+Test-Command: NOMEMTEST=1 make --directory=Release test
+Features: test-name=siridb-unit-tests
+Depends: @, @builddeps@
--- /dev/null
+version=4
+opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*)\.tar\.gz%siridb-server-$1.tar.gz%" \
+ https://github.com/transceptor-technology/siridb-server/releases \
+ (?:.*?/)?v?(\d[\d.]*)\.tar\.gz
--- /dev/null
+FROM alpine:latest
+RUN apk update && \
+ apk upgrade && \
+ apk add gcc make libuv-dev musl-dev pcre2-dev yajl-dev util-linux-dev linux-headers git && \
+ git clone https://github.com/transceptor-technology/libcleri.git /tmp/libcleri && \
+ cd /tmp/libcleri/Release && \
+ make all && \
+ make install && \
+ git clone https://github.com/SiriDB/siridb-server.git /tmp/siridb-server && \
+ cd /tmp/siridb-server/Release && \
+ make clean && \
+ make
+
+FROM alpine:latest
+RUN apk update && \
+ apk add pcre2 libuv libuuid yajl && \
+ mkdir -p /etc/siridb && \
+ mkdir -p /var/lib/siridb
+COPY --from=0 /tmp/siridb-server/Release/siridb-server /usr/local/bin/
+COPY --from=0 /usr/lib/libcleri* /usr/lib/
+
+# Data
+VOLUME ["/var/lib/siridb/"]
+# Client (Socket) connections
+EXPOSE 9000
+# Server (Socket) connections
+EXPOSE 9010
+# Client (HTTP) connections
+EXPOSE 9080
+# Status connection
+EXPOSE 8080
+
+# Overwrite default configuration parameters
+ENV SIRIDB_BIND_SERVER_ADDRESS 0.0.0.0
+ENV SIRIDB_BIND_CLIENT_ADDRESS 0.0.0.0
+ENV SIRIDB_HTTP_API_PORT 9080
+ENV SIRIDB_HTTP_STATUS_PORT 8080
+ENV SIRIDB_ENABLE_SHARD_COMPRESSION 1
+ENV SIRIDB_ENABLE_SHARD_AUTO_DURATION 1
+ENV SIRIDB_BUFFER_SYNC_INTERVAL 500
+
+ENTRYPOINT ["/usr/local/bin/siridb-server"]
--- /dev/null
+# Running SiriDB in Kubernetes
+
+This folder contains a Service, StatefulSet and a PodDisruptionBudget definition which can be used as example configuration
+for running SiriDB in Kubernetes.
+
--- /dev/null
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+ name: siridb-pdb
+spec:
+ maxUnavailable: 1
+ selector:
+ matchLabels:
+ app: siridb
--- /dev/null
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app: siridb
+ name: siridb
+spec:
+ clusterIP: None
+ publishNotReadyAddresses: true
+ ports:
+ - name: status
+ port: 8080
+ - name: client
+ port: 9000
+ - name: http
+ port: 9080
+ - name: server
+ port: 9010
+ selector:
+ app: siridb
\ No newline at end of file
--- /dev/null
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: siridb
+ labels:
+ app: siridb
+spec:
+ selector:
+ matchLabels:
+ app: siridb
+ serviceName: siridb
+ replicas: 2 # Multiple of 2, to create pools with two servers.
+ updateStrategy:
+ type: RollingUpdate
+ podManagementPolicy: Parallel
+ template:
+ metadata:
+ labels:
+ app: siridb
+ spec:
+ terminationGracePeriodSeconds: 120
+ dnsConfig:
+ searches:
+ - siridb.default.svc.cluster.local
+ containers:
+ - name: siridb
+ image: siridb/siridb-server:2.0.42 # Pin to a specific version
+ imagePullPolicy: Always
+ args: ["--managed"] # Tells SiriDB it will be managed by Kubernetes
+ env:
+ - name: SIRIDB_HTTP_STATUS_PORT
+ value: "8080"
+ - name: SIRIDB_HTTP_API_PORT
+ value: "9080"
+ - name: SIRIDB_ENABLE_SHARD_COMPRESSION
+ value: "1"
+ - name: SIRIDB_ENABLE_SHARD_AUTO_DURATION
+ value: "1"
+ - name: SIRIDB_BUFFER_SYNC_INTERVAL
+ value: "500"
+ - name: SIRIDB_DEFAULT_DB_PATH
+ value: /mnt/siridb/
+ - name: SIRIDB_BIND_SERVER_ADDRESS
+ value: "0.0.0.0"
+ - name: SIRIDB_BIND_CLIENT_ADDRESS
+ value: "0.0.0.0"
+ - name: SIRIDB_SERVER_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ ports:
+ - name: status
+ containerPort: 8080
+ - name: client
+ containerPort: 9000
+ - name: http
+ containerPort: 9080
+ - name: server
+ containerPort: 9010
+ volumeMounts:
+ - name: data
+ mountPath: /mnt/siridb/
+ resources:
+ requests:
+ memory: 100M # For example, 3Gi for large data sets
+ livenessProbe:
+ httpGet:
+ path: /healthy
+ port: 8080
+ periodSeconds: 20
+ timeoutSeconds: 10
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: 8080
+ initialDelaySeconds: 15
+ periodSeconds: 20
+ timeoutSeconds: 10
+ volumeClaimTemplates:
+ - metadata:
+ name: data
+ spec:
+ accessModes: ["ReadWriteOnce"]
+ resources:
+ requests:
+ storage: 200Mi # For example, 300Gi for large data sets
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<diagram program="umlet" version="14.2">
+ <zoom_level>9</zoom_level>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>684</x>
+ <y>414</y>
+ <w>207</w>
+ <h>432</h>
+ </coordinates>
+ <panel_attributes>siridb_t
+--
+uuid: uuid_t
+tz: iso8601_tz_t
+shard_mask_num: uint16_t
+shard_mask_log: uint16_t
+buffer_size: size_t
+buffer_len: size_t
+start_ts: uint32_t
+max_series_id: uint32_t
+duration_num: uint64_t
+duration_log: uint64_t
+dbname: *char
+dbpath: *char
+buffer_path: *char
+time: *siridb_time_t
+server: *siridb_server_t
+replica: *siridb_server_t
+users: *siridb_users_t
+servers: *siridb_servers_t
+pools: *siridb_pools_t
+series: *ct_node_t
+series_map: *imap64_t
+shards: *imap64_t
+buffer_fp: *FILE
+--
+siridb_new(): *siridb_t
+siridb_free(): void
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>0</x>
+ <y>414</y>
+ <w>207</w>
+ <h>126</h>
+ </coordinates>
+ <panel_attributes>siri_t
+--
+loop: *uv_loop_t
+grammar: *cleri_grammar_t
+siridb_list: *siridb_list_t
+fh: *siri_fh_t
+--
+siri_start(): int
+siri_free(): void
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>198</x>
+ <y>477</y>
+ <w>153</w>
+ <h>36</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>288</x>
+ <y>378</y>
+ <w>99</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m2=0..*
+group=1</panel_attributes>
+ <additional_attributes>90.0;40.0;90.0;10.0;10.0;10.0;10.0;70.0;50.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>333</x>
+ <y>414</y>
+ <w>207</w>
+ <h>108</h>
+ </coordinates>
+ <panel_attributes>siridb_list_t
+--
+siridb: *siridb_t
+next: *siridb_list_t
+--
+siridb_list_new(): *siridb_list_t
+siridb_list_free(): void
+siridb_get(): *siridb_t
+group=1</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>531</x>
+ <y>477</y>
+ <w>171</w>
+ <h>36</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>10.0;10.0;170.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>0</x>
+ <y>612</y>
+ <w>207</w>
+ <h>126</h>
+ </coordinates>
+ <panel_attributes>siri_fh_t
+--
+size: uint16_t
+idx: uint16_t
+fpointers: **siri_fp_t
+--
+siri_fh_new(): *siri_fh_t
+siri_fh_free(): void
+siri_fopen(): int
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>0</x>
+ <y>873</y>
+ <w>207</w>
+ <h>99</h>
+ </coordinates>
+ <panel_attributes>siri_fp_t
+--
+fp: *FILE
+ref: uint8_t
+--
+siri_fp_new(): *siri_fp_t
+siri_fp_decref(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>99</x>
+ <y>531</y>
+ <w>27</w>
+ <h>99</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<->
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>99</x>
+ <y>729</y>
+ <w>63</w>
+ <h>162</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<->
+m1=1
+m2=0..size</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>306</x>
+ <y>612</y>
+ <w>261</w>
+ <h>144</h>
+ </coordinates>
+ <panel_attributes>siridb_shard_t
+--
+id: uint64_t
+tp: uint8_t
+status: uint8_t
+fp: *siri_fp_t
+--
+siridb_shard_create(): *siridb_shard_t
+siridb_shard_load(): int
+siridb_shard_free(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>558</x>
+ <y>630</y>
+ <w>144</w>
+ <h>36</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>140.0;10.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>198</x>
+ <y>747</y>
+ <w>180</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>180.0;10.0;180.0;170.0;10.0;170.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>198</x>
+ <y>621</y>
+ <w>126</w>
+ <h>36</h>
+ </coordinates>
+ <panel_attributes>lt=<-
+siri_fopen</panel_attributes>
+ <additional_attributes>10.0;20.0;120.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1017</x>
+ <y>414</y>
+ <w>279</w>
+ <h>153</h>
+ </coordinates>
+ <panel_attributes>siridb_users_t
+--
+user: *siridb_user_t
+next: *siridb_users_t
+--
+siridb_users_load(): *siridb_users_t
+siridb_users_free(): void
+siridb_users_add_user(): int
+siridb_users_drop_user(): int
+siridb_users_get_user(): *siridb_user_t
+siridb_users_save(): int
+group=2</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>972</x>
+ <y>378</y>
+ <w>99</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m2=0..*
+group=2</panel_attributes>
+ <additional_attributes>90.0;40.0;90.0;10.0;10.0;10.0;10.0;70.0;50.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>882</x>
+ <y>477</y>
+ <w>153</w>
+ <h>36</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1422</x>
+ <y>414</y>
+ <w>279</w>
+ <h>144</h>
+ </coordinates>
+ <panel_attributes>siridb_user_t
+--
+username: *char
+password: *char
+access_bit: siridb_access_t
+--
+siridb_user_new(): *siridb_user_t
+siridb_user_free(): void
+siridb_user_prop(): void
+siridb_user_set_password(): int
+siridb_user_check_access(): int</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1287</x>
+ <y>477</y>
+ <w>153</w>
+ <h>36</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1017</x>
+ <y>621</y>
+ <w>369</w>
+ <h>216</h>
+ </coordinates>
+ <panel_attributes>siridb_series_t
+--
+id: uint32_t
+tp: uint8_t
+mask: uint16_t
+buffer: *siridb_buffer_t
+index: *siridb_series_idx_t
+--
+siridb_series_load(): int
+siridb_series_new(): *siridb_series_t
+siridb_series_free(): void
+siridb_series_add_idx_num32(): void
+siridb_series_add_idx_num64(): void
+siridb_series_add_point(): void
+siridb_series_get_points_num32(): *siridb_points_t</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>882</x>
+ <y>648</y>
+ <w>153</w>
+ <h>36</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>126</x>
+ <y>765</y>
+ <w>225</w>
+ <h>72</h>
+ </coordinates>
+ <panel_attributes>both siri_fh_t and siridb_shard_t
+can hold a reference to siri_fp_t,
+the last decref will actually
+destroy the object.
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1017</x>
+ <y>909</y>
+ <w>369</w>
+ <h>99</h>
+ </coordinates>
+ <panel_attributes>siridb_series_idx_t
+--
+len: uint32_t
+has_overlap: uint8_t
+idx: *void
+--
+Methods are provided by siridb_series_t</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1179</x>
+ <y>828</y>
+ <w>27</w>
+ <h>99</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1215</x>
+ <y>1080</y>
+ <w>279</w>
+ <h>126</h>
+ </coordinates>
+ <panel_attributes>idx_num64_t
+--
+start_ts: uint64_t
+end_ts: uint64_t
+shard: *siridb_shard_t
+pos: uint32_t
+len: uint16_t
+--
+Methods are provided by siridb_series_t</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1278</x>
+ <y>999</y>
+ <w>63</w>
+ <h>99</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..len</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>909</x>
+ <y>1080</y>
+ <w>279</w>
+ <h>126</h>
+ </coordinates>
+ <panel_attributes>idx_num32_t
+--
+start_ts: uint32_t
+end_ts: uint32_t
+shard: *siridb_shard_t
+pos: uint32_t
+len: uint16_t
+--
+Methods are provided by siridb_series_t</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1098</x>
+ <y>999</y>
+ <w>63</w>
+ <h>99</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..len</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>486</x>
+ <y>747</y>
+ <w>639</w>
+ <h>522</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>690.0;510.0;690.0;560.0;10.0;560.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>450</x>
+ <y>747</y>
+ <w>864</w>
+ <h>549</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>940.0;510.0;940.0;590.0;10.0;590.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>783</x>
+ <y>1008</y>
+ <w>225</w>
+ <h>63</h>
+ </coordinates>
+ <panel_attributes>idx can be used with either
+idx_num32_t or idx_num64_t,
+not both.
+
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>576</x>
+ <y>180</y>
+ <w>315</w>
+ <h>126</h>
+ </coordinates>
+ <panel_attributes>siridb_pools_t
+--
+size: uint16_t
+lookup: *siridb_lookup_t
+pool: *siridb_pool_t
+--
+siridb_pools_init(): void
+siridb_pools_free(): void
+-siridb_pools_gen_lookup(): *siridb_lookup_t</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>747</x>
+ <y>297</y>
+ <w>27</w>
+ <h>135</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>10.0;130.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>576</x>
+ <y>0</y>
+ <w>315</w>
+ <h>99</h>
+ </coordinates>
+ <panel_attributes>siridb_pool_t
+--
+size: uint16_t
+server: *siridb_server_t[2]
+--
+siridb_lookup_sn(): uint16_t
+siridb_lookup_sn_raw(): uint16_t</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>747</x>
+ <y>90</y>
+ <w>63</w>
+ <h>108</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..size</panel_attributes>
+ <additional_attributes>10.0;100.0;10.0;10.0</additional_attributes>
+ </element>
+</diagram>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<diagram program="umlet" version="14.2">
+ <zoom_level>10</zoom_level>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2420</x>
+ <y>590</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=->
+</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>2410</x>
+ <y>370</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2440</x>
+ <y>380</y>
+ <w>170</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0;150.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2420</x>
+ <y>400</y>
+ <w>30</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2540</x>
+ <y>490</y>
+ <w>70</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>50.0;10.0;50.0;90.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2420</x>
+ <y>700</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2540</x>
+ <y>790</y>
+ <w>70</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;50.0;10.0;50.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2420</x>
+ <y>810</y>
+ <w>30</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;130.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2540</x>
+ <y>890</y>
+ <w>70</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;50.0;70.0;50.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2170</x>
+ <y>1030</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_user_object</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2420</x>
+ <y>970</y>
+ <w>30</w>
+ <h>170</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;150.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2210</x>
+ <y>680</y>
+ <w>130</w>
+ <h>280</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<illegal
+password></panel_attributes>
+ <additional_attributes>10.0;260.0;10.0;10.0;110.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2210</x>
+ <y>970</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1710</x>
+ <y>590</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>1700</x>
+ <y>370</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1710</x>
+ <y>400</y>
+ <w>30</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1710</x>
+ <y>700</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1830</x>
+ <y>790</y>
+ <w>70</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;50.0;10.0;50.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1710</x>
+ <y>810</y>
+ <w>30</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;130.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1830</x>
+ <y>890</y>
+ <w>70</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;50.0;70.0;50.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1710</x>
+ <y>970</y>
+ <w>30</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1480</x>
+ <y>790</y>
+ <w>180</w>
+ <h>170</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+error changing user</panel_attributes>
+ <additional_attributes>10.0;150.0;10.0;10.0;130.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1480</x>
+ <y>970</y>
+ <w>150</w>
+ <h>130</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;110.0;130.0;110.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>2410</x>
+ <y>1220</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=flow_final</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2320</x>
+ <y>1120</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_query</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2350</x>
+ <y>1060</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2420</x>
+ <y>1150</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>1700</x>
+ <y>1150</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=flow_final</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1610</x>
+ <y>1060</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_query</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1710</x>
+ <y>1090</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>970</x>
+ <y>370</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1000</x>
+ <y>380</y>
+ <w>170</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0;150.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>980</x>
+ <y>400</y>
+ <w>30</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>970</x>
+ <y>940</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=flow_final</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>880</x>
+ <y>850</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_query</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>980</x>
+ <y>880</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1100</x>
+ <y>690</y>
+ <w>70</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;50.0;70.0;50.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1060</x>
+ <y>490</y>
+ <w>110</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>90.0;10.0;90.0;40.0;10.0;40.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1060</x>
+ <y>590</y>
+ <w>110</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;40.0;90.0;40.0;90.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>980</x>
+ <y>770</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>980</x>
+ <y>590</y>
+ <w>30</w>
+ <h>170</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;150.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2130</x>
+ <y>570</y>
+ <w>210</w>
+ <h>390</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<access denied>
+<cluster offline>
+fg=blue</panel_attributes>
+ <additional_attributes>10.0;370.0;10.0;10.0;190.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1420</x>
+ <y>570</y>
+ <w>210</w>
+ <h>390</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<access denied>
+<cluster offline>
+<use does not exist></panel_attributes>
+ <additional_attributes>10.0;370.0;10.0;10.0;190.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>420</x>
+ <y>370</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>450</x>
+ <y>380</y>
+ <w>170</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0;150.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>430</x>
+ <y>400</y>
+ <w>30</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>420</x>
+ <y>940</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=flow_final</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>330</x>
+ <y>850</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_query</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>430</x>
+ <y>880</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>550</x>
+ <y>690</y>
+ <w>70</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;50.0;70.0;50.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>510</x>
+ <y>490</y>
+ <w>110</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>90.0;10.0;90.0;40.0;10.0;40.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>510</x>
+ <y>590</y>
+ <w>110</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;40.0;90.0;40.0;90.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>430</x>
+ <y>770</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>430</x>
+ <y>590</y>
+ <w>30</w>
+ <h>170</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;150.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>170</x>
+ <y>770</y>
+ <w>180</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;100.0;160.0;100.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>170</x>
+ <y>570</y>
+ <w>180</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+access denied</panel_attributes>
+ <additional_attributes>10.0;170.0;10.0;10.0;160.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>50</x>
+ <y>0</y>
+ <w>670</w>
+ <h>240</h>
+ </coordinates>
+ <panel_attributes>These diagrams are describing how queries are parsed.
+
+Each diagram starts with an initial state where the start time
+is already set and time-expressions are already parsed and translated
+to positive integer values.
+
+All enter and exit functions can be found in "siri/parser/listener.c"
+
+When we talk about the "master" server we are actually talking about
+the server who received the query. This server has the "master" role
+for processing the query.
+fontsize=16
+bg=gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>430</x>
+ <y>1940</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial
+group=1</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>460</x>
+ <y>1950</y>
+ <w>170</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=->
+group=1</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0;150.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>440</x>
+ <y>1970</y>
+ <w>30</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=->
+group=1</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>560</x>
+ <y>2060</y>
+ <w>70</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=->
+group=1</panel_attributes>
+ <additional_attributes>50.0;10.0;50.0;90.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>520</x>
+ <y>2160</y>
+ <w>30</w>
+ <h>250</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;230.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>200</x>
+ <y>290</y>
+ <w>400</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>show statement
+--
+This statement will be answered by the master only.
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>810</x>
+ <y>290</y>
+ <w>400</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>calculator (for time) statement
+--
+This statement will be answered by the master only.
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>1480</x>
+ <y>290</y>
+ <w>400</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>alter user statement
+--
+Can be used to change a users password.
+All servers must process this query.
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>2190</x>
+ <y>290</y>
+ <w>400</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>create user statement
+--
+Can be used to change a users password.
+All servers must process this query.
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>820</x>
+ <y>2280</y>
+ <w>280</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>Probably we can align query_xxx_t
+and create xxx functions which
+can be used by all statements.
+bg=blue</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>680</x>
+ <y>2350</y>
+ <w>220</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>200.0;10.0;10.0;40.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2130</x>
+ <y>970</y>
+ <w>210</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>lt=->
+fg=blue</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;170.0;190.0;170.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>520</x>
+ <y>2420</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>470</x>
+ <y>2160</y>
+ <w>30</w>
+ <h>350</h>
+ </coordinates>
+ <panel_attributes>lt=->
+</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;330.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1720</x>
+ <y>380</y>
+ <w>180</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;160.0;10.0;160.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1790</x>
+ <y>490</y>
+ <w>110</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>90.0;10.0;90.0;40.0;10.0;40.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>420</x>
+ <y>2160</y>
+ <w>30</w>
+ <h>450</h>
+ </coordinates>
+ <panel_attributes>lt=->
+</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;430.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>470</x>
+ <y>2520</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>620</x>
+ <y>2420</y>
+ <w>100</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>80.0;10.0;80.0;190.0;10.0;190.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>370</x>
+ <y>2160</y>
+ <w>30</w>
+ <h>550</h>
+ </coordinates>
+ <panel_attributes>lt=->
+</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;530.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>560</x>
+ <y>2420</y>
+ <w>170</w>
+ <h>310</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>150.0;10.0;150.0;290.0;10.0;290.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>470</x>
+ <y>2520</y>
+ <w>200</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>180.0;10.0;180.0;140.0;10.0;140.0;10.0;170.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>420</x>
+ <y>2620</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>560</x>
+ <y>2800</y>
+ <w>70</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;50.0;70.0;50.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>340</x>
+ <y>2940</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>query_list_free</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>440</x>
+ <y>2880</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>210</x>
+ <y>2880</y>
+ <w>150</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0;130.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>430</x>
+ <y>3130</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=flow_final</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>340</x>
+ <y>3030</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_query</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>440</x>
+ <y>2970</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>440</x>
+ <y>3060</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>150</x>
+ <y>2880</y>
+ <w>210</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>lt=->
+fg=blue</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;170.0;190.0;170.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>520</x>
+ <y>2720</y>
+ <w>110</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;30.0;90.0;30.0;90.0;50.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>440</x>
+ <y>2720</y>
+ <w>30</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;130.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>150</x>
+ <y>2140</y>
+ <w>210</w>
+ <h>730</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<access denied>
+fg=blue</panel_attributes>
+ <additional_attributes>10.0;710.0;10.0;10.0;190.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>210</x>
+ <y>2500</y>
+ <w>260</w>
+ <h>370</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;350.0;10.0;10.0;240.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>230</x>
+ <y>2600</y>
+ <w>190</w>
+ <h>270</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;250.0;10.0;10.0;170.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>210</x>
+ <y>1860</y>
+ <w>400</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>list users statement
+--
+This statement will be answered by the master only.
+
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>340</x>
+ <y>2850</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_send_query_result</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>500</x>
+ <y>2770</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2480</x>
+ <y>860</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1770</x>
+ <y>860</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1040</x>
+ <y>660</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>490</x>
+ <y>660</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2320</x>
+ <y>940</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_send_query_result</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1610</x>
+ <y>940</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_send_query_result</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>880</x>
+ <y>740</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_send_query_result</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>330</x>
+ <y>740</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_send_query_result</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>80</x>
+ <y>2850</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_query_send_error</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2060</x>
+ <y>940</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_query_send_error</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1350</x>
+ <y>940</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_query_send_error</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>70</x>
+ <y>740</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_query_send_error</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>340</x>
+ <y>2690</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_list_users_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>400</x>
+ <y>2590</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_limit_expr
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>450</x>
+ <y>2490</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_where_user_stmt
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>500</x>
+ <y>2390</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_user_columns</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>340</x>
+ <y>2130</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_list_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>500</x>
+ <y>2030</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_timeit_stmt
+group=1</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2480</x>
+ <y>460</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1770</x>
+ <y>460</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1040</x>
+ <y>460</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>490</x>
+ <y>460</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2320</x>
+ <y>560</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_create_user_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2320</x>
+ <y>670</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_set_password_expr</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2320</x>
+ <y>780</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_create_user_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1610</x>
+ <y>560</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_alter_user_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1610</x>
+ <y>780</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_alter_user_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1610</x>
+ <y>670</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_set_password_expr</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>880</x>
+ <y>560</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_calc_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>330</x>
+ <y>560</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_show_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>670</x>
+ <y>2350</y>
+ <w>340</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>320.0;10.0;100.0;130.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3330</x>
+ <y>560</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_grant_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>3420</x>
+ <y>370</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3450</x>
+ <y>380</y>
+ <w>170</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0;150.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3430</x>
+ <y>400</y>
+ <w>30</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3550</x>
+ <y>490</y>
+ <w>70</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>50.0;10.0;50.0;90.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3490</x>
+ <y>460</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>3190</x>
+ <y>290</y>
+ <w>410</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>grant statement
+--
+Can be used to grant access rights to a user or network.
+All servers must process this query.
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>2100</x>
+ <y>450</y>
+ <w>270</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>bind new "user_object" to query
+when successful.
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2210</x>
+ <y>500</y>
+ <w>170</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>140</x>
+ <y>2020</y>
+ <w>270</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>binds "query_list_t" to query when
+successful.
+(must be destroyed)
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>250</x>
+ <y>2070</y>
+ <w>170</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1500</x>
+ <y>500</y>
+ <w>170</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3430</x>
+ <y>590</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3090</x>
+ <y>950</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_set_password_expr</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3160</x>
+ <y>1060</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_grant_user_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3210</x>
+ <y>980</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3430</x>
+ <y>690</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>3420</x>
+ <y>750</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=decision</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3260</x>
+ <y>750</y>
+ <w>180</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=->
+grant user</panel_attributes>
+ <additional_attributes>160.0;20.0;10.0;20.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3450</x>
+ <y>750</y>
+ <w>180</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=->
+grant network</panel_attributes>
+ <additional_attributes>10.0;20.0;160.0;20.0;160.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3330</x>
+ <y>660</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_access_expr</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3300</x>
+ <y>630</y>
+ <w>80</w>
+ <h>50</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;60.0;30.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>3110</x>
+ <y>620</y>
+ <w>200</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>bind children to query
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3160</x>
+ <y>840</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_grant_user_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3120</x>
+ <y>780</y>
+ <w>120</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;100.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>3010</x>
+ <y>730</y>
+ <w>240</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>replace query->children with
+existing "user_object"
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3210</x>
+ <y>870</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3350</x>
+ <y>870</y>
+ <w>30</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;190.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3580</x>
+ <y>950</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_set_comment_expr
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3490</x>
+ <y>1060</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_grant_network_stmt
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3650</x>
+ <y>980</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3490</x>
+ <y>840</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_grant_network_stmt
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3650</x>
+ <y>870</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3540</x>
+ <y>870</y>
+ <w>30</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;190.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>3630</x>
+ <y>730</y>
+ <w>250</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>replace query->children with
+existing "network_object"
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3670</x>
+ <y>780</y>
+ <w>100</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>80.0;10.0;10.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3330</x>
+ <y>1170</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3350</x>
+ <y>1090</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3510</x>
+ <y>1090</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3430</x>
+ <y>1200</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3090</x>
+ <y>1300</y>
+ <w>260</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0;240.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>3420</x>
+ <y>1430</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=flow_final</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3330</x>
+ <y>1350</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_query</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3430</x>
+ <y>1380</y>
+ <w>30</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;50.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>3330</x>
+ <y>1260</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_send_query_result</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>2910</x>
+ <y>1270</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_query_send_error</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3210</x>
+ <y>1090</y>
+ <w>140</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>120.0;190.0;10.0;190.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3550</x>
+ <y>1090</y>
+ <w>80</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;190.0;60.0;190.0;60.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3430</x>
+ <y>1290</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2960</x>
+ <y>570</y>
+ <w>390</w>
+ <h>720</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<access denied>
+<cluster offline></panel_attributes>
+ <additional_attributes>10.0;700.0;10.0;10.0;370.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3050</x>
+ <y>960</y>
+ <w>130</w>
+ <h>330</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<illegal
+new password>
+</panel_attributes>
+ <additional_attributes>10.0;310.0;10.0;10.0;40.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3010</x>
+ <y>850</y>
+ <w>170</w>
+ <h>440</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<user does
+not exist></panel_attributes>
+ <additional_attributes>10.0;420.0;10.0;10.0;150.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>1390</x>
+ <y>450</y>
+ <w>270</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>bind existing "user_object" to
+query when successful.
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2960</x>
+ <y>850</y>
+ <w>940</w>
+ <h>740</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<network does not exist></panel_attributes>
+ <additional_attributes>10.0;460.0;10.0;720.0;920.0;720.0;920.0;10.0;760.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2240</x>
+ <y>790</y>
+ <w>180</w>
+ <h>170</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<error saving users></panel_attributes>
+ <additional_attributes>10.0;150.0;10.0;10.0;80.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3090</x>
+ <y>1070</y>
+ <w>130</w>
+ <h>220</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<error saving
+users></panel_attributes>
+ <additional_attributes>10.0;200.0;10.0;10.0;70.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3010</x>
+ <y>960</y>
+ <w>860</w>
+ <h>600</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<illegal comment></panel_attributes>
+ <additional_attributes>10.0;350.0;10.0;580.0;840.0;580.0;840.0;10.0;800.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4580</x>
+ <y>560</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_revoke_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>4670</x>
+ <y>370</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4700</x>
+ <y>380</y>
+ <w>170</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0;150.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4680</x>
+ <y>400</y>
+ <w>30</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4800</x>
+ <y>490</y>
+ <w>70</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>50.0;10.0;50.0;90.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4740</x>
+ <y>460</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>4440</x>
+ <y>290</y>
+ <w>410</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>revoke statement
+--
+Can be used to grant access rights to a user or network.
+All servers must process this query.
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4680</x>
+ <y>590</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4410</x>
+ <y>940</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_revoke_user_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4680</x>
+ <y>690</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>4670</x>
+ <y>750</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=decision</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4510</x>
+ <y>750</y>
+ <w>180</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=->
+grant user</panel_attributes>
+ <additional_attributes>160.0;20.0;10.0;20.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4700</x>
+ <y>750</y>
+ <w>180</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=->
+grant network</panel_attributes>
+ <additional_attributes>10.0;20.0;160.0;20.0;160.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4580</x>
+ <y>660</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_access_expr</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4550</x>
+ <y>630</y>
+ <w>80</w>
+ <h>50</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;60.0;30.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>4340</x>
+ <y>620</y>
+ <w>220</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>bind children to query
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4410</x>
+ <y>840</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_revoke_user_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4350</x>
+ <y>780</y>
+ <w>140</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;120.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>4240</x>
+ <y>730</y>
+ <w>270</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>replace query->children with
+existing "user_object"
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4510</x>
+ <y>870</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4740</x>
+ <y>940</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_revoke network_stmt
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4740</x>
+ <y>840</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_revoke_network_stmt
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4850</x>
+ <y>870</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>4870</x>
+ <y>730</y>
+ <w>270</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>replace query->children with
+existing "network_object"
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4920</x>
+ <y>780</y>
+ <w>90</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>70.0;10.0;10.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4580</x>
+ <y>1050</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4600</x>
+ <y>970</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4760</x>
+ <y>970</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4680</x>
+ <y>1080</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4340</x>
+ <y>1180</y>
+ <w>260</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0;240.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>4670</x>
+ <y>1320</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=flow_final</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4580</x>
+ <y>1230</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_query</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4680</x>
+ <y>1260</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4580</x>
+ <y>1140</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_send_query_result</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>4160</x>
+ <y>1150</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_query_send_error</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4460</x>
+ <y>970</y>
+ <w>140</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>120.0;190.0;10.0;190.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4800</x>
+ <y>970</y>
+ <w>80</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;190.0;60.0;190.0;60.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4680</x>
+ <y>1170</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4210</x>
+ <y>570</y>
+ <w>390</w>
+ <h>600</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<access denied>
+<cluster offline></panel_attributes>
+ <additional_attributes>10.0;580.0;10.0;10.0;370.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4260</x>
+ <y>850</y>
+ <w>170</w>
+ <h>320</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<user does
+not exist></panel_attributes>
+ <additional_attributes>10.0;300.0;10.0;10.0;150.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4210</x>
+ <y>850</y>
+ <w>940</w>
+ <h>600</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<network does not exist></panel_attributes>
+ <additional_attributes>10.0;340.0;10.0;580.0;920.0;580.0;920.0;10.0;760.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4340</x>
+ <y>950</y>
+ <w>130</w>
+ <h>220</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<error saving
+users></panel_attributes>
+ <additional_attributes>10.0;200.0;10.0;10.0;70.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>3050</x>
+ <y>1070</y>
+ <w>790</w>
+ <h>460</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<error saving networks></panel_attributes>
+ <additional_attributes>10.0;240.0;10.0;440.0;770.0;440.0;770.0;10.0;670.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>4260</x>
+ <y>950</y>
+ <w>830</w>
+ <h>470</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<error saving networks></panel_attributes>
+ <additional_attributes>10.0;240.0;10.0;450.0;810.0;450.0;810.0;10.0;710.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>2560</x>
+ <y>720</y>
+ <w>150</w>
+ <h>50</h>
+ </coordinates>
+ <panel_attributes>reset free call
+when successful.
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>2520</x>
+ <y>760</y>
+ <w>100</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>80.0;10.0;10.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>380</x>
+ <y>3380</y>
+ <w>170</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=->
+group=2</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;10.0;150.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>360</x>
+ <y>3400</y>
+ <w>30</w>
+ <h>180</h>
+ </coordinates>
+ <panel_attributes>lt=->
+group=2</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;160.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>480</x>
+ <y>3490</y>
+ <w>70</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=->
+group=2</panel_attributes>
+ <additional_attributes>50.0;10.0;50.0;90.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>420</x>
+ <y>3460</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_timeit_stmt
+group=2</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>350</x>
+ <y>3370</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial
+group=2</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>390</x>
+ <y>3590</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=->
+</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>390</x>
+ <y>3690</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>290</x>
+ <y>3590</y>
+ <w>30</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>lt=->
+</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;170.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>480</x>
+ <y>3870</y>
+ <w>70</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;50.0;70.0;50.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>260</x>
+ <y>4010</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>query_count_free</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>360</x>
+ <y>3950</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>130</x>
+ <y>3950</y>
+ <w>150</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0;130.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>350</x>
+ <y>4200</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=flow_final</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>260</x>
+ <y>4100</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>free_query</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>360</x>
+ <y>4040</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>360</x>
+ <y>4130</y>
+ <w>30</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;70.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>70</x>
+ <y>3950</y>
+ <w>210</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>lt=->
+fg=blue</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;170.0;190.0;170.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>440</x>
+ <y>3790</y>
+ <w>110</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;30.0;90.0;30.0;90.0;50.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>360</x>
+ <y>3790</y>
+ <w>30</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;130.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>70</x>
+ <y>3570</y>
+ <w>210</w>
+ <h>370</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<access denied>
+fg=blue</panel_attributes>
+ <additional_attributes>10.0;350.0;10.0;10.0;190.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>130</x>
+ <y>3290</y>
+ <w>400</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>count users statement
+--
+This statement will be answered by the master only.
+
+bg=light_gray</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>260</x>
+ <y>3920</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_send_query_result</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>420</x>
+ <y>3840</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>0</x>
+ <y>3920</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_query_send_error</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>260</x>
+ <y>3760</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_count_users_stmt
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>370</x>
+ <y>3660</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_where_xxx_stmt
+bg=orange</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>260</x>
+ <y>3560</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_count_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>60</x>
+ <y>3450</y>
+ <w>270</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>binds "query_count_t" to query when
+successful.
+(must be destroyed)
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>170</x>
+ <y>3500</y>
+ <w>170</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;150.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>130</x>
+ <y>3770</y>
+ <w>150</w>
+ <h>170</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-</panel_attributes>
+ <additional_attributes>10.0;150.0;10.0;10.0;130.0;10.0</additional_attributes>
+ </element>
+</diagram>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<diagram program="umlet" version="14.2">
+ <help_text>// Uncomment the following line to change the fontsize and font:
+fontsize=13
+fontfamily=Monospaced //possible: SansSerif,Serif,
+
+
+//////////////////////////////////////////////////////////////////////////////////////////////
+// Welcome to UMLet!
+//
+// Double-click on elements to add them to the diagram, or to copy them
+// Edit elements by modifying the text in this panel
+// Hold Ctrl to select multiple elements
+// Use Ctrl+mouse to select via lasso
+//
+// Use +/- or Ctrl+mouse wheel to zoom
+// Drag a whole relation at its central square icon
+//
+// Press Ctrl+C to copy the whole diagram to the system clipboard (then just paste it to, eg, Word)
+// Edit the files in the "palettes" directory to create your own element palettes
+//
+// Select "Custom Elements > New..." to create new element types
+//////////////////////////////////////////////////////////////////////////////////////////////
+
+
+// This text will be stored with each diagram; use it for notes.</help_text>
+ <zoom_level>8</zoom_level>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>448</x>
+ <y>712</y>
+ <w>200</w>
+ <h>216</h>
+ </coordinates>
+ <panel_attributes>siri_t
+--
+args: *siri_args_t
+backup: *uv_timer_t
+cfg: *siri_cfg_t
+fh: *siri_fh_t
+grammar: *cleri_grammar_t
+heartbeat: *uv_timer_t
+loop: * uv_loop_t
+optimize: *siri_optimize_t
+siridb_list: *llist_t
+siridb_mutex: uv_mutex_t
+startup_time: uint32_t
+status: siri_status_t
+--
+siri_start(): int
+siri_free(): void
+siri_setup_logger(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>752</x>
+ <y>712</y>
+ <w>224</w>
+ <h>632</h>
+ </coordinates>
+ <panel_attributes>siridb_t
+--
+active_tasks: uint16_t
+buffer_fp: *FILE
+buffer_len: size_t
+buffer_path: * char
+buffer_size: size_t
+dbname: * char
+dbpath: * char
+drop_threshold: double
+dropped_fp: *FILE
+duration_log: uint64_t
+duration_num: uint64_t
+fifo: *siridb_fifo_t
+flags: uint8_t
+groups: *siridb_groups_t
+index_size: size_t
+insert_tasks: uint16_t
+max_series_id: uint32_t
+pools: *siridb_pools_t
+received_points: size_t
+ref: uint16_t
+reindex: *siridb_reindex_t
+replica: * siridb_server_t
+replicate: *siridb_replicate_t
+series: *ct_t
+series_map: *imap_t
+series_mutex: uv_mutex_t
+server: * siridb_server_t
+servers: *llist_t
+shard_mask_log: uint16_t
+shard_mask_num: uint16_t
+shards: *imap_t
+shards_mutex: uv_mutex_t
+start_ts: time_t
+store: *qp_fpacker_t
+time: * siridb_time_t
+tz: iso8601_tz_t
+users: *llist_t
+uuid: uuid_t
+--
+siridb__free(): void
+siridb_decref(): void
+siridb_decref_cb(): void
+siridb_from_unpacker(): int
+siridb_get(): *siridb_t
+siridb_incref(): void
+siridb_is_db_path(): int
+siridb_is_reindexing(): int
+siridb_new(): *siridb_t
+siridb_open_files(): int
+siridb_save(): int
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>640</x>
+ <y>816</y>
+ <w>128</w>
+ <h>32</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>10.0;10.0;140.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1048</x>
+ <y>712</y>
+ <w>296</w>
+ <h>192</h>
+ </coordinates>
+ <panel_attributes>lt=.
+servers
+--
+siridb_servers_available(): int
+siridb_servers_by_name(): *siridb_server_t
+siridb_servers_by_uuid(): *siridb_server_t
+siridb_servers_free(): void
+siridb_servers_get_file(): ssize_t
+siridb_servers_list(): int
+siridb_servers_load(): int
+siridb_servers_online(): int
+siridb_servers_other2vec(): *vec_t
+siridb_servers_register(): int
+siridb_servers_save(): int
+siridb_servers_send_flags(): void
+siridb_servers_send_pkg(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1432</x>
+ <y>408</y>
+ <w>288</w>
+ <h>560</h>
+ </coordinates>
+ <panel_attributes>server_t
+--
+address: *char
+buffer_path: *char
+buffer_size: size_t
+dbpath: *char
+flags: uint8_t
+id: uint8_t
+ip_support: uint8_t
+libuv: *char
+name: *char
+pid: uint16_t
+pool: uint16_t
+port: uint16_t
+promises: *imap_t
+ref: uint16_t
+socket: *uv_tcp_t
+startup_time: uint32_t
+uuid: uuid_t
+version: *char
+--
+siridb__server_free(): void
+siridb_server_cexpr_cb(): int
+siridb_server_cmp(): int
+siridb_server_connect(): void
+siridb_server_decref(): void
+siridb_server_drop(): int
+siridb_server_from_node(): *siridb_server_t
+siridb_server_incref(): void
+siridb_server_is_accessible(): int
+siridb_server_is_available(): int
+siridb_server_is_connected(): int
+siridb_server_is_online(): int
+siridb_server_is_remote_prop(): int
+siridb_server_is_synchronizing(): int
+siridb_server_new(): *siridb_server_t
+siridb_server_register(): *siridb_server_t
+siridb_server_self_accessible(): int
+siridb_server_self_available(): int
+siridb_server_self_online(): int
+siridb_server_self_synchronizing(): int
+siridb_server_send_flags(): void
+siridb_server_send_pkg(): int
+siridb_server_str_status(): *char
+siridb_server_update_address(): int
+siridb_server_update_flags(): void
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>968</x>
+ <y>920</y>
+ <w>480</w>
+ <h>32</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1..*</panel_attributes>
+ <additional_attributes>10.0;10.0;580.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>968</x>
+ <y>800</y>
+ <w>96</w>
+ <h>24</h>
+ </coordinates>
+ <panel_attributes>lt=.</panel_attributes>
+ <additional_attributes>10.0;10.0;100.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1336</x>
+ <y>800</y>
+ <w>112</w>
+ <h>24</h>
+ </coordinates>
+ <panel_attributes>lt=.</panel_attributes>
+ <additional_attributes>120.0;10.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>504</x>
+ <y>544</y>
+ <w>200</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>iso8601_tz_t
+--
+tz: iso8601_tz_t
+--
+iso8601_tz(): iso8601_tz_t
+iso8601_tzname(): const *char
+iso8601_parse_date(): int64_t
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>448</x>
+ <y>992</y>
+ <w>200</w>
+ <h>112</h>
+ </coordinates>
+ <panel_attributes>siri_fh_t
+--
+size: uint16_t
+idx: uint16_t
+fpointers: **siri_fp_t
+--
+siri_fh_new(): *siri_fh_t
+siri_fh_free(): void
+siri_fopen(): int
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>448</x>
+ <y>1176</y>
+ <w>200</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>siri_fp_t
+--
+fp: *FILE
+ref: uint8_t
+--
+siri_fp_new(): *siri_fp_t
+siri_fp_decref(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>536</x>
+ <y>920</y>
+ <w>24</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>536</x>
+ <y>1096</y>
+ <w>64</w>
+ <h>96</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..size</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;100.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>736</x>
+ <y>1528</y>
+ <w>288</w>
+ <h>352</h>
+ </coordinates>
+ <panel_attributes>siridb_shard_t
+--
+flags: uint8_t
+fn: *char
+fp: *siri_fp_t
+id: uint64_t
+max_chunk_sz: uint16_t
+ref: uint32_t
+replacing: *siridb_shard_t
+tp: uint8_t
+--
+siridb__shard_decref(): void
+siridb__shard_free(): void
+siridb_shard_cexpr_cb(): int
+siridb_shard_create(): *siridb_shard_t
+siridb_shard_decref(): void
+siridb_shard_drop(): void
+siridb_shard_get_points_log32(): int
+siridb_shard_get_points_log64(): int
+siridb_shard_get_points_num32(): int
+siridb_shard_get_points_num64(): int
+siridb_shard_get_size(): ssize_t
+siridb_shard_incref(): void
+siridb_shard_load(): int
+siridb_shard_optimize(): int
+siridb_shard_remove(): int
+siridb_shard_status(): void
+siridb_shard_write_flags(): int
+siridb_shard_write_points(): long int</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>784</x>
+ <y>1408</y>
+ <w>256</w>
+ <h>56</h>
+ </coordinates>
+ <panel_attributes>lt=.
+shards
+--
+siridb_shards_add_points(): int
+siridb_shards_load(): int</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>872</x>
+ <y>1336</y>
+ <w>24</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>lt=.</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>872</x>
+ <y>1456</y>
+ <w>24</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>lt=.</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>760</x>
+ <y>1336</y>
+ <w>48</w>
+ <h>208</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;240.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>640</x>
+ <y>1024</y>
+ <w>112</w>
+ <h>584</h>
+ </coordinates>
+ <panel_attributes>lt=<-
+</panel_attributes>
+ <additional_attributes>10.0;10.0;90.0;10.0;90.0;710.0;120.0;710.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>536</x>
+ <y>1256</y>
+ <w>216</w>
+ <h>440</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>250.0;520.0;10.0;520.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1104</x>
+ <y>1048</y>
+ <w>296</w>
+ <h>488</h>
+ </coordinates>
+ <panel_attributes>siridb_series_t
+--
+buffer: *siridb_buffer_t
+end: uint64_t
+flags: uint8_t
+id: uint32_t
+idx: *idx_t
+idx_len: uint32_t
+length: uint32_t
+mask: uint16_t
+name: *char
+name_len: uint16_t
+pool: uint16_t
+ref: uint32_t
+start: uint64_t
+tp: uint8_t
+--
+siridb__series_decref(): void
+siridb__series_free(): void
+siridb_series_add_idx(): int
+siridb_series_add_pcache(): int
+siridb_series_add_point(): int
+siridb_series_cexpr_cb(): int
+siridb_series_decref(): void
+siridb_series_drop(): int
+siridb_series_drop_commit(): int
+siridb_series_drop_prepare(): void
+siridb_series_flush_dropped(): int
+siridb_series_get_points(): *siridb_points_t
+siridb_series_incref(): void
+siridb_series_isnum(): int
+siridb_series_load(): int
+siridb_series_new(): *siridb_series_t
+siridb_series_open_store(): int
+siridb_series_optimize_shard(): int
+siridb_series_remove_shard(): void
+siridb_series_replicate_file(): int
+siridb_series_server_id(): int
+siridb_series_server_id_by_name(): uint8_t
+siridb_series_update_props(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>968</x>
+ <y>1096</y>
+ <w>152</w>
+ <h>32</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>10.0;10.0;170.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1104</x>
+ <y>1608</y>
+ <w>296</w>
+ <h>128</h>
+ </coordinates>
+ <panel_attributes>idx_t
+--
+end_ts: uint64_t
+len: uint16_t
+log_sz: uint16_t
+pos: uint32_t
+shard: *siridb_shard_t
+start_ts: uint64_t
+--
+/functions can be found in/
+/siridb_series_t class/</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1248</x>
+ <y>1528</y>
+ <w>56</w>
+ <h>96</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..len</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;100.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1016</x>
+ <y>1664</y>
+ <w>104</w>
+ <h>32</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+m1=0..*
+m2=1</panel_attributes>
+ <additional_attributes>110.0;10.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1464</x>
+ <y>992</y>
+ <w>256</w>
+ <h>280</h>
+ </coordinates>
+ <panel_attributes>siridb_groups_t
+--
+flags: uint8_t
+fn: *char
+groups: *ct_t
+mutex: uv_mutex_t
+ngroups: *vec_t
+nseries: *vec_t
+ref: uint8_t
+status: uint8_t
+work: uv_work_t
+--
+siridb_groups_add_group(): int
+siridb_groups_add_series(): void
+siridb_groups_decref(): void
+siridb_groups_destroy(): void
+siridb_groups_drop_group(): int
+siridb_groups_get_file(): ssize_t
+siridb_groups_init_nseries(): void
+siridb_groups_new(): *siridb_groups_t
+siridb_groups_pkg(): *sirinet_pkg_t
+siridb_groups_save(): int
+siridb_groups_start(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>968</x>
+ <y>1016</y>
+ <w>512</w>
+ <h>32</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>10.0;10.0;620.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1464</x>
+ <y>1352</y>
+ <w>256</w>
+ <h>280</h>
+ </coordinates>
+ <panel_attributes>siridb_group_t
+--
+flags: uint16_t
+n: uint32_t
+name: *char
+ref: uint16_t
+regex: *pcre
+regex_extra: *pcre_extra
+series: *vec_t
+source: *char
+--
+siridb__group_decref(): void
+siridb__group_free(): void
+siridb_group_cexpr_cb(): int
+siridb_group_cleanup(): void
+siridb_group_decref(): void
+siridb_group_incref(): void
+siridb_group_is_remote_prop(): int
+siridb_group_new(): *siridb_group_t
+siridb_group_prop(): void
+siridb_group_set_name(): int
+siridb_group_test_series(): int
+siridb_group_update_expression(): int</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1576</x>
+ <y>1264</y>
+ <w>48</w>
+ <h>104</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;110.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1392</x>
+ <y>1304</y>
+ <w>168</w>
+ <h>64</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+m1=0..*
+m2=0..*</panel_attributes>
+ <additional_attributes>160.0;60.0;160.0;10.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>120</x>
+ <y>712</y>
+ <w>200</w>
+ <h>144</h>
+ </coordinates>
+ <panel_attributes>cfg_t
+--
+default_db_path[]: char
+heartbeat_interval: uint16_t
+ip_support: uint8_t
+listen_backend_port: uint16_t
+listen_client_port: uint16_t
+max_open_files: uint16_t
+optimize_interval: uint32_t
+server_address[]: char
+--
+siri_cfg_init(): void
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>312</x>
+ <y>760</y>
+ <w>152</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=-()
+m2=cfg file
+</panel_attributes>
+ <additional_attributes>170.0;20.0;10.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>600</x>
+ <y>624</y>
+ <w>184</w>
+ <h>104</h>
+ </coordinates>
+ <panel_attributes>lt=-()
+m2=tz translation
+</panel_attributes>
+ <additional_attributes>210.0;110.0;210.0;60.0;10.0;60.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1096</x>
+ <y>456</y>
+ <w>248</w>
+ <h>216</h>
+ </coordinates>
+ <panel_attributes>siridb_fifo_t
+--
+path: *char
+fifos: *llist_t
+in: *siridb_ffile_t
+out: *siridb_ffile_t
+max_id: ssize_t
+--
+siridb_fifo_append(): int
+siridb_fifo_close(): int
+siridb_fifo_commit(): int
+siridb_fifo_commit_err(): int
+siridb_fifo_free(): void
+siridb_fifo_has_data(): uint32_t
+siridb_fifo_is_open(): int
+siridb_fifo_new(): *siridb_fifo_t
+siridb_fifo_open(): int
+siridb_fifo_pop(): *sirinet_pkg_t
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>944</x>
+ <y>640</y>
+ <w>168</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..1</panel_attributes>
+ <additional_attributes>10.0;90.0;10.0;10.0;190.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>120</x>
+ <y>920</y>
+ <w>200</w>
+ <h>96</h>
+ </coordinates>
+ <panel_attributes>args_t
+--
+config[]: char
+log_colorized: int32_t
+log_level[]: char
+version: int32_t
+--
+siri_args_parse(): void
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>208</x>
+ <y>848</y>
+ <w>56</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+config</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>312</x>
+ <y>856</y>
+ <w>152</w>
+ <h>128</h>
+ </coordinates>
+ <panel_attributes>lt=-()
+m2=startup arguments
+</panel_attributes>
+ <additional_attributes>170.0;10.0;90.0;10.0;90.0;130.0;10.0;130.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1808</x>
+ <y>832</y>
+ <w>280</w>
+ <h>192</h>
+ </coordinates>
+ <panel_attributes>siridb_pools_t
+--
+len: uint16_t
+pool: *siridb_pool_t
+lookup: *siridb_lookup_t
+prev_lookup: *siridb_lookup_t
+--
+siridb_pools_accessible(): int
+siridb_pools_append(): *siridb_pool_t
+siridb_pools_available(): int
+siridb_pools_free(): void
+siridb_pools_gen_lookup(): *siridb_lookup_t
+siridb_pools_init(): void
+siridb_pools_online(): int
+siridb_pools_send_pkg(): void
+siridb_pools_send_pkg_2some(): void
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>968</x>
+ <y>968</y>
+ <w>856</w>
+ <h>32</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1</panel_attributes>
+ <additional_attributes>10.0;10.0;1050.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1808</x>
+ <y>576</y>
+ <w>280</w>
+ <h>192</h>
+ </coordinates>
+ <panel_attributes>siridb_pool_t
+--
+len: uint16_t
+server[2]: *siridb_server_t
+--
+siridb_pool_accessible(): int
+siridb_pool_add_server(): void
+siridb_pool_available(): int
+siridb_pool_cexpr_cb(): int
+siridb_pool_online(): int
+siridb_pool_send_pkg(): int
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1944</x>
+ <y>760</y>
+ <w>48</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1..*</panel_attributes>
+ <additional_attributes>10.0;90.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1712</x>
+ <y>664</y>
+ <w>112</w>
+ <h>32</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+m1=1
+m2=1..len</panel_attributes>
+ <additional_attributes>120.0;10.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1096</x>
+ <y>160</y>
+ <w>248</w>
+ <h>216</h>
+ </coordinates>
+ <panel_attributes>siridb_ffile_t
+--
+fd: int
+fn: *char
+fp: *FILE
+free_space: uint32_t
+id: uint64_t
+next_size: uint32_t
+size: long int
+--
+siridb_ffile_append(): siridb_ffile_result_t
+siridb_ffile_check_fn(): int
+siridb_ffile_free(): void
+siridb_ffile_new(): *siridb_ffile_t
+siridb_ffile_open(): void
+siridb_ffile_pop(): *sirinet_pkg_t
+siridb_ffile_pop_commit(): int
+siridb_ffile_unlink(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1224</x>
+ <y>368</y>
+ <w>48</w>
+ <h>104</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1..*</panel_attributes>
+ <additional_attributes>10.0;110.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>808</x>
+ <y>456</y>
+ <w>256</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=.
+users
+--
+siridb_users_add_user(): int
+siridb_users_drop_user(): int
+siridb_users_free(): void
+siridb_users_get_file(): ssize_t
+siridb_users_get_user(): *siridb_user_t
+siridb_users_load(): int
+siridb_users_save(): *int </panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>744</x>
+ <y>184</y>
+ <w>232</w>
+ <h>192</h>
+ </coordinates>
+ <panel_attributes>siridb_user_t
+--
+ref: uint16_t
+access_bit: uint32_t
+name: *char
+password: *char
+--
+siridb_user_new(): *siridb_user_t
+siridb_user_prop(): void
+siridb_user_set_name(): int
+siridb_user_set_password(): int
+siridb_user_check_access(): int
+siridb__user_free(): void
+siridb_user_cexpr_cb(): int
+siridb_user_incref(): void
+siridb_user_decref(): void</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>776</x>
+ <y>368</y>
+ <w>48</w>
+ <h>360</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=0..*</panel_attributes>
+ <additional_attributes>10.0;430.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>856</x>
+ <y>568</y>
+ <w>24</w>
+ <h>160</h>
+ </coordinates>
+ <panel_attributes>lt=.</panel_attributes>
+ <additional_attributes>10.0;180.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>856</x>
+ <y>368</y>
+ <w>24</w>
+ <h>104</h>
+ </coordinates>
+ <panel_attributes>lt=.</panel_attributes>
+ <additional_attributes>10.0;110.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1432</x>
+ <y>104</y>
+ <w>288</w>
+ <h>232</h>
+ </coordinates>
+ <panel_attributes>sirinet_socket_t
+--
+buf: *char
+len: size_t
+on_data: on_data_cb_t
+origin: *void
+ref: uint32_t
+siridb: *siridb_t
+tcp: uv_tcp_t
+tp: sirinet_socket_tp_t
+--
+sirinet__socket_free(): void
+sirinet_addr_and_port(): int
+sirinet_socket_alloc_buffer(): void
+sirinet_socket_decref(): void
+sirinet_socket_incref(): void
+sirinet_socket_ip_support_str(): const *char
+sirinet_socket_new(): *uv_tcp_t
+sirinet_socket_on_data(): void
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1576</x>
+ <y>328</y>
+ <w>24</w>
+ <h>96</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<->>>>
+m2=1
+</panel_attributes>
+ <additional_attributes>10.0;100.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>968</x>
+ <y>128</y>
+ <w>480</w>
+ <h>128</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+m1=0..origin
+m2=0..*
+</panel_attributes>
+ <additional_attributes>10.0;130.0;90.0;130.0;90.0;10.0;580.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>120</x>
+ <y>544</y>
+ <w>200</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>lt=.
+clserver
+--
+sirinet_clserver_init(): int
+--
+/responsible for client /
+/connections and requests/</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>120</x>
+ <y>416</y>
+ <w>200</w>
+ <h>88</h>
+ </coordinates>
+ <panel_attributes>lt=.
+bserver
+--
+sirinet_bserver_init(): int
+--
+/responsible for server /
+/connections and requests/</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>312</x>
+ <y>32</y>
+ <w>1296</w>
+ <h>456</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1..*</panel_attributes>
+ <additional_attributes>10.0;540.0;90.0;540.0;90.0;10.0;1570.0;10.0;1570.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>312</x>
+ <y>48</y>
+ <w>1232</w>
+ <h>568</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+m1=1
+m2=1..*</panel_attributes>
+ <additional_attributes>10.0;680.0;110.0;680.0;110.0;10.0;1490.0;10.0;1490.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>72</x>
+ <y>584</y>
+ <w>64</w>
+ <h>192</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+config</panel_attributes>
+ <additional_attributes>60.0;10.0;10.0;10.0;10.0;220.0;60.0;220.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>32</x>
+ <y>448</y>
+ <w>104</w>
+ <h>368</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+config</panel_attributes>
+ <additional_attributes>110.0;10.0;10.0;10.0;10.0;440.0;110.0;440.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>312</x>
+ <y>608</y>
+ <w>200</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-</panel_attributes>
+ <additional_attributes>230.0;130.0;230.0;90.0;40.0;90.0;40.0;10.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>312</x>
+ <y>480</y>
+ <w>216</w>
+ <h>248</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-</panel_attributes>
+ <additional_attributes>250.0;290.0;250.0;230.0;60.0;230.0;60.0;10.0;10.0;10.0</additional_attributes>
+ </element>
+</diagram>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<diagram program="umlet" version="14.2">
+ <zoom_level>10</zoom_level>
+ <element>
+ <id>UMLSpecialState</id>
+ <coordinates>
+ <x>810</x>
+ <y>60</y>
+ <w>40</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>type=initial</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>720</x>
+ <y>190</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_alter_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>820</x>
+ <y>90</y>
+ <w>30</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=-></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;100.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>320</x>
+ <y>770</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>siridb_query_send_error</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>410</x>
+ <y>200</y>
+ <w>330</w>
+ <h>590</h>
+ </coordinates>
+ <panel_attributes>lt=<<<-
+<no access></panel_attributes>
+ <additional_attributes>10.0;570.0;10.0;10.0;310.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>720</x>
+ <y>360</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_alter_group</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>980</x>
+ <y>360</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_alter_user</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1240</x>
+ <y>360</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_alter_server</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1490</x>
+ <y>360</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_alter_servers</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1740</x>
+ <y>360</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_set_drop_threshold</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>1990</x>
+ <y>360</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_set_timezone</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>720</x>
+ <y>470</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_set_expression</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>860</x>
+ <y>610</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>enter_set_name</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLState</id>
+ <coordinates>
+ <x>970</x>
+ <y>1170</y>
+ <w>230</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>exit_timeit_stmt</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>620</x>
+ <y>120</y>
+ <w>160</w>
+ <h>30</h>
+ </coordinates>
+ <panel_attributes>bind <q_alter>
+bg=yellow</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>710</x>
+ <y>140</y>
+ <w>80</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=..</panel_attributes>
+ <additional_attributes>10.0;10.0;60.0;50.0</additional_attributes>
+ </element>
+</diagram>
--- /dev/null
+#!/usr/bin/env python
+'''Export python grammar to C grammar files
+
+Author: Jeroen van der Heijden (Transceptor Technology)
+Date: 2016-10-10
+'''
+import sys
+import os
+from grammar import siri_grammar
+from pyleri import Grammar
+
+
+if __name__ == '__main__':
+ c_file, h_file = siri_grammar.export_c(target='siri/grammar/grammar')
+
+ EXPORT_PATH = 'cgrammar'
+
+ try:
+ os.makedirs(EXPORT_PATH)
+ except FileExistsError:
+ pass
+
+ with open(os.path.join(EXPORT_PATH, 'grammar.c'),
+ 'w',
+ encoding='utf-8') as f:
+ f.write(c_file)
+ with open(os.path.join(EXPORT_PATH, 'grammar.h'),
+ 'w',
+ encoding='utf-8') as f:
+ f.write(h_file)
+
+ print('\nFinished creating new c-grammar files...\n')
+
+ EXPORT_PATH = 'jsgrammar'
+
+ try:
+ os.makedirs(EXPORT_PATH)
+ except FileExistsError:
+ pass
+
+ js_file = siri_grammar.export_js(js_template=Grammar.JS_WINDOW_TEMPLATE)
+
+ with open(os.path.join(EXPORT_PATH, 'grammar.js'),
+ 'w',
+ encoding='utf-8') as f:
+ f.write(js_file)
+
+ js_es6_file = siri_grammar.export_js()
+
+ with open(os.path.join(EXPORT_PATH, 'SiriGrammar.js'),
+ 'w',
+ encoding='utf-8') as f:
+ f.write(js_es6_file)
+
+ print('\nFinished creating new js-grammar files...\n')
+
+ py_file = siri_grammar.export_py()
+
+ EXPORT_PATH = 'pygrammar'
+
+ try:
+ os.makedirs(EXPORT_PATH)
+ except FileExistsError:
+ pass
+
+ with open(os.path.join(EXPORT_PATH, 'grammar.py'),
+ 'w',
+ encoding='utf-8') as f:
+ f.write(py_file)
+
+ print('\nFinished creating new py-grammar file...\n')
+
+ go_file = siri_grammar.export_go()
+
+ EXPORT_PATH = 'gogrammar'
+
+ try:
+ os.makedirs(EXPORT_PATH)
+ except FileExistsError:
+ pass
+
+ with open(os.path.join(EXPORT_PATH, 'grammar.go'),
+ 'w',
+ encoding='utf-8') as f:
+ f.write(go_file)
+
+ print('\nFinished creating new go-grammar file...\n')
+
+ java_file = siri_grammar.export_java()
+
+ EXPORT_PATH = 'javagrammar'
+
+ try:
+ os.makedirs(EXPORT_PATH)
+ except FileExistsError:
+ pass
+
+ with open(os.path.join(EXPORT_PATH, 'SiriGrammar.java'),
+ 'w',
+ encoding='utf-8') as f:
+ f.write(java_file)
+
+ print('\nFinished creating new java-grammar file...\n')
--- /dev/null
+import logging
+import re
+from siridbhelp import help_structure
+from pyleri import Choice
+from pyleri import Grammar
+from pyleri import Keyword
+from pyleri import List
+from pyleri import Optional
+from pyleri import Prio
+from pyleri import Ref
+from pyleri import Regex
+from pyleri import Repeat
+from pyleri import Sequence
+from pyleri import THIS
+from pyleri import Token
+from pyleri import Tokens
+
+
+class SiriGrammar(Grammar):
+ '''
+ SiriDB grammar.
+
+ Note: choices can be optimized using most_greedy=False when there
+ is a preferable order in choices.
+ This only should be used when there's no conflict in making a
+ decision by the parser. (e.g. two choices should start with the
+ same keyword because in that case we should usually take the most
+ greedy one)
+ '''
+
+ RE_KEYWORDS = re.compile('[a-z_]+')
+
+ # Regular expressions
+ r_float = Regex('[-+]?[0-9]*\.?[0-9]+')
+ r_integer = Regex('[-+]?[0-9]+')
+ r_uinteger = Regex('[0-9]+')
+ r_time_str = Regex('[0-9]+[smhdw]')
+ r_singleq_str = Regex('(?:\'(?:[^\']*)\')+')
+ r_doubleq_str = Regex('(?:"(?:[^"]*)")+')
+ r_grave_str = Regex('(?:`(?:[^`]*)`)+')
+ r_uuid_str = Regex(
+ '[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12}')
+ # we only allow an optional 'i' for case-insensitive regex
+ r_regex = Regex('(/[^/\\\\]*(?:\\\\.[^/\\\\]*)*/i?)')
+ r_comment = Regex('#.*')
+
+ # Keywords
+ k_access = Keyword('access')
+ k_active_handles = Keyword('active_handles')
+ k_active_tasks = Keyword('active_tasks')
+ k_address = Keyword('address')
+ k_after = Keyword('after')
+ k_all = Keyword('all')
+ k_alter = Keyword('alter')
+ k_and = Keyword('and')
+ k_as = Keyword('as')
+ k_backup_mode = Keyword('backup_mode')
+ k_before = Keyword('before')
+ k_buffer_size = Keyword('buffer_size')
+ k_buffer_path = Keyword('buffer_path')
+ k_between = Keyword('between')
+ k_count = Keyword('count')
+ k_create = Keyword('create')
+ k_critical = Keyword('critical')
+ k_database = Keyword('database')
+ k_dbname = Keyword('dbname')
+ k_dbpath = Keyword('dbpath')
+ k_debug = Keyword('debug')
+ k_derivative = Keyword('derivative')
+ k_difference = Keyword('difference')
+ k_drop = Keyword('drop')
+ k_drop_threshold = Keyword('drop_threshold')
+ k_duration_log = Keyword('duration_log')
+ k_duration_num = Keyword('duration_num')
+ k_end = Keyword('end')
+ k_error = Keyword('error')
+ k_expiration_log = Keyword('expiration_log')
+ k_expiration_num = Keyword('expiration_num')
+ k_expression = Keyword('expression')
+ k_false = Keyword('false')
+ k_fifo_files = Keyword('fifo_files')
+ k_filter = Keyword('filter')
+ k_first = Keyword('first')
+ k_float = Keyword('float')
+ k_for = Keyword('for')
+ k_from = Keyword('from')
+ k_full = Keyword('full')
+ k_grant = Keyword('grant')
+ k_group = Keyword('group')
+ k_groups = Keyword('groups')
+ k_help = Choice(Keyword('help'), Token('?'))
+ k_idle_percentage = Keyword('idle_percentage')
+ k_idle_time = Keyword('idle_time')
+ k_inf = Keyword('inf')
+ k_info = Keyword('info')
+ k_ignore_threshold = Keyword('ignore_threshold')
+ k_insert = Keyword('insert')
+ k_integer = Keyword('integer')
+ k_intersection = Choice(
+ Token('&'),
+ Keyword('intersection'),
+ most_greedy=False)
+ k_interval = Keyword('interval')
+ k_ip_support = Keyword('ip_support')
+ k_last = Keyword('last')
+ k_length = Keyword('length')
+ k_libuv = Keyword('libuv')
+ k_limit = Keyword('limit')
+ k_list = Keyword('list')
+ k_list_limit = Keyword('list_limit')
+ k_log = Keyword('log')
+ k_log_level = Keyword('log_level')
+ k_max = Keyword('max')
+ k_max_open_files = Keyword('max_open_files')
+ k_mean = Keyword('mean')
+ k_median = Keyword('median')
+ k_median_high = Keyword('median_high')
+ k_median_low = Keyword('median_low')
+ k_mem_usage = Keyword('mem_usage')
+ k_merge = Keyword('merge')
+ k_min = Keyword('min')
+ k_modify = Keyword('modify')
+ k_name = Keyword('name')
+ k_nan = Keyword('nan')
+ k_ninf = Sequence('-', k_inf)
+ k_now = Keyword('now')
+ k_number = Keyword('number')
+ k_online = Keyword('online')
+ k_open_files = Keyword('open_files')
+ k_or = Keyword('or')
+ k_password = Keyword('password')
+ k_points = Keyword('points')
+ k_pool = Keyword('pool')
+ k_pools = Keyword('pools')
+ k_port = Keyword('port')
+ k_prefix = Keyword('prefix')
+ k_pvariance = Keyword('pvariance')
+ k_read = Keyword('read')
+ k_received_points = Keyword('received_points')
+ k_reindex_progress = Keyword('reindex_progress')
+ k_revoke = Keyword('revoke')
+ k_select = Keyword('select')
+ k_select_points_limit = Keyword('select_points_limit')
+ k_selected_points = Keyword('selected_points')
+ k_series = Keyword('series')
+ k_server = Keyword('server')
+ k_servers = Keyword('servers')
+ k_set = Keyword('set')
+ k_shard_duration = Keyword('shard_duration')
+ k_shards = Keyword('shards')
+ k_show = Keyword('show')
+ k_sid = Keyword('sid')
+ k_size = Keyword('size')
+ k_start = Keyword('start')
+ k_startup_time = Keyword('startup_time')
+ k_status = Keyword('status')
+ k_stddev = Keyword('stddev')
+ k_string = Keyword('string')
+ k_suffix = Keyword('suffix')
+ k_sum = Keyword('sum')
+ k_symmetric_difference = Choice(
+ Token('^'),
+ Keyword('symmetric_difference'),
+ most_greedy=False)
+ k_sync_progress = Keyword('sync_progress')
+ k_tag = Keyword('tag')
+ k_tags = Keyword('tags')
+ k_tee_pipe_name = Keyword('tee_pipe_name')
+ k_time_precision = Keyword('time_precision')
+ k_timeit = Keyword('timeit')
+ k_timeval = Keyword('timeval')
+ k_timezone = Keyword('timezone')
+ k_to = Keyword('to')
+ k_true = Keyword('true')
+ k_type = Keyword('type')
+ k_union = Choice(
+ Tokens(', |'),
+ Keyword('union'),
+ most_greedy=False)
+ k_untag = Keyword('untag')
+ k_uptime = Keyword('uptime')
+ k_user = Keyword('user')
+ k_users = Keyword('users')
+ k_using = Keyword('using')
+ k_uuid = Keyword('uuid')
+ k_variance = Keyword('variance')
+ k_version = Keyword('version')
+ k_warning = Keyword('warning')
+ k_where = Keyword('where')
+ k_who_am_i = Keyword('who_am_i')
+ k_write = Keyword('write')
+ c_difference = Choice(
+ Token('-'),
+ k_difference,
+ most_greedy=False)
+
+ access_keywords = Choice(
+ k_read,
+ k_write,
+ k_modify,
+ k_full,
+ k_select,
+ k_show,
+ k_list,
+ k_count,
+ k_create,
+ k_insert,
+ k_drop,
+ k_grant,
+ k_revoke,
+ k_alter,
+ most_greedy=False)
+
+ _boolean = Choice(k_true, k_false, most_greedy=False)
+
+ log_keywords = Choice(
+ k_debug,
+ k_info,
+ k_warning,
+ k_error,
+ k_critical,
+ most_greedy=False)
+
+ int_expr = Prio(
+ r_integer,
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, Tokens('+ - * % /'), THIS))
+
+ string = Choice(r_singleq_str, r_doubleq_str, most_greedy=False)
+
+ time_expr = Prio(
+ r_time_str,
+ k_now,
+ string,
+ r_integer,
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, Tokens('+ - * % /'), THIS))
+
+ series_columns = List(Choice(
+ k_name,
+ k_type,
+ k_length,
+ k_start,
+ k_end,
+ k_shard_duration,
+ k_pool,
+ most_greedy=False), ',', 1)
+
+ shard_columns = List(Choice(
+ k_sid,
+ k_pool,
+ k_server,
+ k_size,
+ k_start,
+ k_end,
+ k_type,
+ k_status,
+ most_greedy=False), ',', 1)
+
+ server_columns = List(Choice(
+ # Local properties
+ k_address,
+ k_buffer_path,
+ k_buffer_size,
+ k_dbpath,
+ k_ip_support,
+ k_libuv,
+ k_name,
+ k_port,
+ k_uuid,
+ k_pool,
+ k_version,
+ k_online,
+ k_startup_time,
+ k_status,
+ # Remote properties
+ k_active_handles,
+ k_active_tasks,
+ k_fifo_files,
+ k_idle_percentage,
+ k_idle_time,
+ k_log_level,
+ k_max_open_files,
+ k_mem_usage,
+ k_open_files,
+ k_received_points,
+ k_reindex_progress,
+ k_selected_points,
+ k_sync_progress,
+ k_tee_pipe_name,
+ k_uptime,
+ most_greedy=False), ',', 1)
+
+ group_columns = List(Choice(
+ k_expression,
+ k_name,
+ k_series,
+ most_greedy=False), ',', 1)
+
+ user_columns = List(Choice(
+ k_name,
+ k_access,
+ most_greedy=False), ',', 1)
+
+ tag_columns = List(Choice(
+ k_name,
+ k_series,
+ most_greedy=False), ',', 1)
+
+ pool_props = Choice(
+ k_pool,
+ k_servers,
+ k_series,
+ most_greedy=False)
+ pool_columns = List(pool_props, ',', 1)
+
+ bool_operator = Tokens('== !=')
+ int_operator = Tokens('< > == != <= >=')
+ str_operator = Tokens('< > == != <= >= ~ !~')
+
+ # where group
+ where_group = Sequence(k_where, Prio(
+ Sequence(k_series, int_operator, int_expr),
+ Sequence(
+ Choice(k_expression, k_name, most_greedy=False),
+ str_operator,
+ string),
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, k_and, THIS),
+ Sequence(THIS, k_or, THIS)))
+
+ # where tag
+ where_tag = Sequence(k_where, Prio(
+ Sequence(k_name, str_operator, string),
+ Sequence(k_series, int_operator, int_expr),
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, k_and, THIS),
+ Sequence(THIS, k_or, THIS)))
+
+ # where pool
+ where_pool = Sequence(k_where, Prio(
+ Sequence(pool_props, int_operator, int_expr),
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, k_and, THIS),
+ Sequence(THIS, k_or, THIS)))
+
+ # where series
+ where_series = Sequence(k_where, Prio(
+ Sequence(
+ Choice(k_length, k_pool, most_greedy=False),
+ int_operator,
+ int_expr),
+ Sequence(k_name, str_operator, string),
+ Sequence(
+ Choice(k_start, k_end, k_shard_duration, most_greedy=False),
+ int_operator,
+ time_expr),
+ Sequence(
+ k_type,
+ bool_operator,
+ Choice(k_string, k_integer, k_float, most_greedy=False)),
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, k_and, THIS),
+ Sequence(THIS, k_or, THIS)))
+
+ # where server
+ where_server = Sequence(k_where, Prio(
+ Sequence(Choice(
+ k_active_handles,
+ k_active_tasks,
+ k_buffer_size,
+ k_fifo_files,
+ k_idle_percentage,
+ k_idle_time,
+ k_port,
+ k_pool,
+ k_startup_time,
+ k_max_open_files,
+ k_mem_usage,
+ k_open_files,
+ k_received_points,
+ k_selected_points,
+ k_uptime,
+ most_greedy=False), int_operator, int_expr),
+ Sequence(Choice(
+ k_address,
+ k_buffer_path,
+ k_dbpath,
+ k_ip_support,
+ k_libuv,
+ k_name,
+ k_uuid,
+ k_version,
+ k_status,
+ k_reindex_progress,
+ k_sync_progress,
+ k_tee_pipe_name,
+ most_greedy=False), str_operator, string),
+ Sequence(k_online, bool_operator, _boolean),
+ Sequence(k_log_level, int_operator, log_keywords),
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, k_and, THIS),
+ Sequence(THIS, k_or, THIS)))
+
+ # where shard
+ where_shard = Sequence(k_where, Prio(
+ Sequence(
+ Choice(k_sid, k_pool, k_size, most_greedy=False),
+ int_operator,
+ int_expr),
+ Sequence(Choice(k_server, k_status), str_operator, string),
+ Sequence(
+ Choice(k_start, k_end, most_greedy=False),
+ int_operator,
+ time_expr),
+ Sequence(
+ k_type,
+ bool_operator,
+ Choice(k_number, k_log, most_greedy=False)),
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, k_and, THIS),
+ Sequence(THIS, k_or, THIS)))
+
+ # where user
+ where_user = Sequence(k_where, Prio(
+ Sequence(k_name, str_operator, string),
+ Sequence(k_access, int_operator, access_keywords),
+ Sequence('(', THIS, ')'),
+ Sequence(THIS, k_and, THIS),
+ Sequence(THIS, k_or, THIS)))
+
+ series_setopr = Choice(
+ k_union,
+ c_difference,
+ k_intersection,
+ k_symmetric_difference,
+ most_greedy=False)
+
+ series_parentheses = Sequence('(', THIS, ')')
+
+ series_all = Choice(Token('*'), k_all, most_greedy=False)
+ series_name = Repeat(string, 1, 1)
+ group_name = Repeat(r_grave_str, 1, 1)
+ tag_name = Repeat(r_grave_str, 1, 1)
+ series_re = Repeat(r_regex, 1, 1)
+ uuid = Choice(r_uuid_str, string, most_greedy=False)
+ group_tag_match = Repeat(r_grave_str, 1, 1)
+ series_match = Prio(
+ List(Choice(
+ series_all,
+ series_name,
+ group_tag_match,
+ series_re,
+ most_greedy=False), series_setopr, 1),
+ Choice(
+ series_all,
+ series_name,
+ group_tag_match,
+ series_re,
+ most_greedy=False),
+ series_parentheses,
+ Sequence(THIS, series_setopr, THIS),
+ )
+
+ limit_expr = Sequence(k_limit, int_expr)
+
+ before_expr = Sequence(k_before, time_expr)
+ after_expr = Sequence(k_after, time_expr)
+ between_expr = Sequence(k_between, time_expr, k_and, time_expr)
+ access_expr = List(access_keywords, ',', 1)
+
+ prefix_expr = Sequence(k_prefix, string)
+ suffix_expr = Sequence(k_suffix, string)
+
+ f_all = Choice(Token('*'), k_all, most_greedy=False)
+
+ f_points = Repeat(k_points, 1, 1) # DEPRECATED
+
+ f_difference = Sequence(
+ k_difference,
+ '(',
+ Optional(time_expr),
+ ')')
+ f_derivative = Sequence(
+ k_derivative,
+ '(',
+ List(time_expr, ',', 0, 2),
+ ')')
+ f_mean = Sequence(
+ k_mean,
+ '(', Optional(time_expr), ')')
+ f_median = Sequence(
+ k_median,
+ '(', Optional(time_expr), ')')
+ f_median_low = Sequence(
+ k_median_low,
+ '(', Optional(time_expr), ')')
+ f_median_high = Sequence(
+ k_median_high,
+ '(', Optional(time_expr), ')')
+ f_sum = Sequence(
+ k_sum,
+ '(', Optional(time_expr), ')')
+ f_min = Sequence(
+ k_min,
+ '(', Optional(time_expr), ')')
+ f_max = Sequence(
+ k_max,
+ '(', Optional(time_expr), ')')
+ f_count = Sequence(
+ k_count,
+ '(', Optional(time_expr), ')')
+ f_variance = Sequence(
+ k_variance,
+ '(', Optional(time_expr), ')')
+ f_pvariance = Sequence(
+ k_pvariance,
+ '(', Optional(time_expr), ')')
+ f_stddev = Sequence(
+ k_stddev,
+ '(', Optional(time_expr), ')')
+ f_first = Sequence(
+ k_first,
+ '(', Optional(time_expr), ')')
+ f_last = Sequence(
+ k_last,
+ '(', Optional(time_expr), ')')
+ f_timeval = Sequence(
+ k_timeval,
+ '(', ')')
+ f_interval = Sequence(
+ k_interval,
+ '(', ')')
+
+ f_filter = Sequence(
+ k_filter,
+ '(',
+ Optional(str_operator),
+ Choice(
+ string,
+ r_integer,
+ r_float,
+ r_regex,
+ k_nan,
+ k_inf,
+ k_ninf,
+ most_greedy=True),
+ ')')
+ f_limit = Sequence(
+ k_limit,
+ '(',
+ int_expr,
+ ',',
+ Choice(
+ k_mean,
+ k_median,
+ k_median_high,
+ k_median_low,
+ k_sum,
+ k_min,
+ k_max,
+ k_count,
+ k_variance,
+ k_pvariance,
+ k_stddev,
+ k_first,
+ k_last,
+ most_greedy=False),
+ ')')
+
+ aggregate_functions = List(Choice(
+ f_all,
+ f_limit,
+ f_mean,
+ f_sum,
+ f_median,
+ f_median_low,
+ f_median_high,
+ f_min,
+ f_max,
+ f_count,
+ f_variance,
+ f_pvariance,
+ f_stddev,
+ f_first,
+ f_last,
+ f_timeval,
+ f_interval,
+ f_difference,
+ f_derivative,
+ f_filter,
+ f_points,
+ most_greedy=False), '=>', 1)
+
+ select_aggregate = Sequence(
+ aggregate_functions,
+ Optional(prefix_expr),
+ Optional(suffix_expr))
+
+ select_aggregates = List(select_aggregate, ',', 1)
+
+ merge_as = Sequence(
+ k_merge,
+ k_as,
+ string,
+ Optional(Sequence(k_using, aggregate_functions)))
+
+ set_address = Sequence(k_set, k_address, string)
+ set_tee_pipe_name = Sequence(k_set, k_tee_pipe_name, Choice(
+ k_false,
+ string,
+ most_greedy=False))
+ set_backup_mode = Sequence(k_set, k_backup_mode, _boolean)
+ set_drop_threshold = Sequence(k_set, k_drop_threshold, r_float)
+ set_expression = Sequence(k_set, k_expression, r_regex)
+ set_ignore_threshold = Sequence(k_set, k_ignore_threshold, _boolean)
+ set_list_limit = Sequence(k_set, k_list_limit, r_uinteger)
+ set_log_level = Sequence(k_set, k_log_level, log_keywords)
+ set_name = Sequence(k_set, k_name, string)
+ set_password = Sequence(k_set, k_password, string)
+ set_port = Sequence(k_set, k_port, r_uinteger)
+ set_select_points_limit = Sequence(
+ k_set, k_select_points_limit, r_uinteger)
+ set_timezone = Sequence(k_set, k_timezone, string)
+ tag_series = Sequence(k_tag, tag_name)
+ untag_series = Sequence(k_untag, tag_name)
+ set_expiration_num = Sequence(
+ k_set,
+ k_expiration_num,
+ time_expr,
+ Optional(set_ignore_threshold))
+ set_expiration_log = Sequence(
+ k_set,
+ k_expiration_log,
+ time_expr,
+ Optional(set_ignore_threshold))
+
+ alter_database = Sequence(k_database, Choice(
+ set_drop_threshold,
+ set_list_limit,
+ set_select_points_limit,
+ set_timezone,
+ set_expiration_num,
+ set_expiration_log,
+ most_greedy=False))
+
+ alter_group = Sequence(k_group, group_name, Choice(
+ set_expression,
+ set_name,
+ most_greedy=False))
+
+ alter_tag = Sequence(k_tag, tag_name, Choice(
+ set_name,
+ most_greedy=False))
+
+ alter_server = Sequence(k_server, uuid, Choice(
+ set_log_level,
+ set_backup_mode,
+ set_tee_pipe_name,
+ set_address,
+ set_port,
+ most_greedy=False))
+
+ alter_servers = Sequence(k_servers, Optional(where_server), Choice(
+ set_log_level,
+ set_tee_pipe_name,
+ most_greedy=False))
+
+ alter_user = Sequence(k_user, string, Choice(
+ set_password,
+ set_name,
+ most_greedy=False))
+
+ alter_series = Sequence(
+ k_series,
+ series_match,
+ Optional(where_series),
+ Choice(tag_series, untag_series, most_greedy=False))
+
+ count_groups = Sequence(
+ k_groups, Optional(where_group))
+ count_tags = Sequence(
+ k_tags, Optional(where_tag))
+ count_pools = Sequence(
+ k_pools, Optional(where_pool))
+ count_series = Sequence(
+ k_series, Optional(series_match), Optional(where_series))
+ count_servers = Sequence(
+ k_servers, Optional(where_server))
+ count_servers_received = Sequence(
+ k_servers,
+ k_received_points,
+ Optional(where_server))
+ count_servers_selected = Sequence(
+ k_servers,
+ k_selected_points,
+ Optional(where_server))
+ count_shards = Sequence(
+ k_shards, Optional(where_shard))
+ count_shards_size = Sequence(
+ k_shards, k_size, Optional(where_shard))
+ count_users = Sequence(
+ k_users, Optional(where_user))
+ count_series_length = Sequence(
+ k_series,
+ k_length,
+ Optional(series_match),
+ Optional(where_series))
+
+ create_group = Sequence(
+ k_group, group_name, k_for, r_regex)
+ create_user = Sequence(
+ k_user, string, set_password)
+
+ drop_group = Sequence(k_group, group_name)
+ drop_tag = Sequence(k_tag, tag_name)
+
+ # Drop statement needs at least a series_math or where STMT or both
+ drop_series = Sequence(
+ k_series,
+ Optional(series_match),
+ Optional(where_series),
+ Optional(set_ignore_threshold))
+ drop_shards = Sequence(
+ k_shards,
+ Optional(where_shard),
+ Optional(set_ignore_threshold))
+ drop_server = Sequence(k_server, uuid)
+ drop_user = Sequence(k_user, string)
+
+ grant_user = Sequence(
+ k_user, string, Optional(set_password))
+
+ list_groups = Sequence(
+ k_groups, Optional(group_columns), Optional(where_group))
+ list_tags = Sequence(
+ k_tags, Optional(tag_columns), Optional(where_tag))
+ list_pools = Sequence(
+ k_pools, Optional(pool_columns), Optional(where_pool))
+ list_series = Sequence(
+ k_series,
+ Optional(series_columns),
+ Optional(series_match),
+ Optional(where_series))
+ list_servers = Sequence(
+ k_servers, Optional(server_columns), Optional(where_server))
+ list_shards = Sequence(
+ k_shards, Optional(shard_columns), Optional(where_shard))
+ list_users = Sequence(
+ k_users, Optional(user_columns), Optional(where_user))
+
+ revoke_user = Sequence(k_user, string)
+
+ alter_stmt = Sequence(k_alter, Choice(
+ alter_series,
+ alter_user,
+ alter_group,
+ alter_tag,
+ alter_server,
+ alter_servers,
+ alter_database,
+ most_greedy=False))
+
+ calc_stmt = Repeat(time_expr, 1, 1)
+
+ count_stmt = Sequence(k_count, Choice(
+ count_groups,
+ count_pools,
+ count_series,
+ count_servers,
+ count_servers_received,
+ count_servers_selected,
+ count_shards,
+ count_shards_size,
+ count_users,
+ count_tags,
+ count_series_length,
+ most_greedy=True))
+
+ create_stmt = Sequence(k_create, Choice(
+ create_group,
+ create_user))
+
+ drop_stmt = Sequence(k_drop, Choice(
+ drop_group,
+ drop_tag,
+ drop_series,
+ drop_shards,
+ drop_server,
+ drop_user,
+ most_greedy=False))
+
+ grant_stmt = Sequence(k_grant, access_expr, k_to, Choice(
+ grant_user,
+ most_greedy=False))
+
+ list_stmt = Sequence(k_list, Choice(
+ list_series,
+ list_tags,
+ list_users,
+ list_shards,
+ list_groups,
+ list_servers,
+ list_pools,
+ most_greedy=False
+ ), Optional(limit_expr))
+
+ revoke_stmt = Sequence(k_revoke, access_expr, k_from, Choice(
+ revoke_user,
+ most_greedy=False))
+
+ select_stmt = Sequence(
+ k_select,
+ select_aggregates,
+ k_from,
+ series_match,
+ Optional(where_series),
+ Optional(Choice(
+ after_expr,
+ between_expr,
+ before_expr,
+ most_greedy=False)),
+ Optional(merge_as))
+
+ show_stmt = Sequence(k_show, List(Choice(
+ k_active_handles,
+ k_active_tasks,
+ k_buffer_path,
+ k_buffer_size,
+ k_dbname,
+ k_dbpath,
+ k_drop_threshold,
+ k_duration_log,
+ k_duration_num,
+ k_fifo_files,
+ k_expiration_log,
+ k_expiration_num,
+ k_idle_percentage,
+ k_idle_time,
+ k_ip_support,
+ k_libuv,
+ k_list_limit,
+ k_log_level,
+ k_max_open_files,
+ k_mem_usage,
+ k_open_files,
+ k_pool,
+ k_received_points,
+ k_reindex_progress,
+ k_selected_points,
+ k_select_points_limit,
+ k_server,
+ k_startup_time,
+ k_status,
+ k_sync_progress,
+ k_tee_pipe_name,
+ k_time_precision,
+ k_timezone,
+ k_uptime,
+ k_uuid,
+ k_version,
+ k_who_am_i,
+ most_greedy=False), ',', 0))
+
+ timeit_stmt = Repeat(k_timeit, 1, 1)
+
+ help_stmt = Ref()
+
+ START = Sequence(
+ Optional(timeit_stmt),
+ Optional(Choice(
+ select_stmt,
+ list_stmt,
+ count_stmt,
+ alter_stmt,
+ create_stmt,
+ drop_stmt,
+ grant_stmt,
+ revoke_stmt,
+ show_stmt,
+ calc_stmt,
+ help_stmt,
+ most_greedy=False)),
+ Optional(r_comment))
+
+ help_stmt = Sequence(k_help) # Dummy
+
+
+def _set_attribute(cls, name, value):
+ setattr(cls, name, value)
+ if name not in cls._order:
+ cls._order.append(name)
+ if hasattr(value, 'name'):
+ raise SyntaxError('Element name is set to {0!r} and therefore cannot '
+ 'be set to {1!r}. Use Repeat({0}, 1, 1) as a '
+ 'workaround.'.format(value.name, name))
+ value.name = name
+
+
+def _walk(cls, path, structure):
+ name = '_'.join(path)
+ try:
+ if structure:
+ for child_path, child in structure.items():
+ _walk(cls, child_path, child)
+
+ opt = Optional(Choice(*[
+ getattr(cls, '_'.join(p)) for p in structure.keys()]))
+
+ value = Sequence(*[getattr(
+ cls,
+ 'k_' + path[-1],
+ Keyword(path[-1])), opt])
+ else:
+ value = Keyword(path[-1])
+ except AttributeError:
+ logging.critical('Cannot parse help file: {!r}'.format(
+ '{}.md'.format('_'.join(path))))
+ import sys
+ sys.exit(1)
+ else:
+ if name != 'help':
+ _set_attribute(cls, name, value)
+ else:
+ # Replace the ref element
+ cls.help_stmt._element = value
+
+
+def _build_help(cls):
+ '''Add 'help' statements to this class.
+
+ Help files are MarkDown files which are read by the help module.
+ '''
+ for path, structure in help_structure.items():
+ _walk(cls, path, structure)
+
+_build_help(SiriGrammar)
+
+siri_grammar = SiriGrammar()
--- /dev/null
+'''Help module.
+
+Build help structure for SiriDB grammar.
+
+:copyright: 2016, Jeroen van der Heijden (Transceptor Technology)
+'''
+
+import os
+import re
+import sys
+
+#
+# Created with ls -A1
+#
+help_files = [f[:-3] for f in os.listdir('../help') if f.endswith('.md')]
+help_files.sort()
+
+def _build_structure(help_files):
+ '''Return a tree structure for the help files.'''
+ _structure = {}
+
+ def _walk(d, keys, i):
+ i += 1
+ k = tuple(keys[:i])
+ if k not in d:
+ d[k] = {}
+ if i < len(keys):
+ _walk(d[k], keys, i)
+ for help_file in help_files:
+ _walk(_structure, help_file.split('_'), 0)
+ return _structure
+
+
+help_structure = _build_structure(help_files)
+
+
+if __name__ == '__main__':
+ print(help_structure)
\ No newline at end of file
--- /dev/null
+help
+====
+
+Displays this help page.
+
+Alias: ?
+
+Syntax:
+
+ help [<option>] [...]
+
+Available help options:
+
+- `timeit`: see `help timeit` for more information.
+- `show`: see `help show` for more information.
+- `count`: see `help count` for more information.
+- `list`: see `help list` for more information.
+- `select`: see `help select` for more information.
+- `create`: see `help create` for more information.
+- `alter`: see `help alter` for more information.
+- `drop`: see `help drop` for more information.
+- `grant`: see `help grant` for more information.
+- `revoke`: see `help revoke` for more information.
+
+help notes
+----------
+Values between [ ] are optional.
+
+Values between < > are not real keywords but explained later in the help file.
+
+Three dots (...) means more.
+
+string values
+-------------
+A string value should be between single-, or double-quotes. When you use a single quote to start the string and need a single quote inside the string you need to prefix the single quote with another single quote.
+
+Examples:
+
+ 'This is a double quote " and a single quote '' in a single quoted string'
+ "This is a double quote "" and a single quote ' in a double quoted string"
+
+time expressions
+----------------
+Time expressions are translated to a time-stamp by SiriDB. When using a 'seconds' based precision database the value should be between 0 and 4294967295 and when 'milliseconds' are used, values between 0 and 1,844674407×10¹⁹ are allowed. In a time expressions it's allowed to use integer values, date strings and keyword `now` for the current time-stamp. Float values are not allowed since we expect time-stamps to be integer values. It's possible to use suffixes 'd', 'h', 'm' and 'ms' behind integer values to specify days, hours, minutes, seconds and \*milliseconds (be careful using milliseconds when having a 'seconds' precision database since the value will be floored to seconds *before* calculating the expression).
+
+The following operators can be used:
+
+ +, -, *, / and %
+
+Note: `/` is a division towards zero, for example: `10 / 3` will return time-stamp value 3
+
+Examples:
+
+ now - 10d # Returns the time-stamp for exactly 10 days ago.
+ '2015-04-01' + 1h * 24 # Returns the time-stamp of April 2, 2015.
+
+date strings
+------------
+Date strings are strings between single- or double-quotes representing a time-stamp value at a given date/time.
+
+Format:
+
+ YYYY-MM-DD HH:MM:SSZ
+
+Instead of a space between date and time it's also possible to use capital T.
+We do not need to specify the whole date/time when another precision is enough.
+These are all valid date strings:
+
+ '2013-02-10 13:04:12Z'
+ '2013-02-10 13:04:12+0200'
+ '2013-2-10T13:04Z'
+ '2013-2-10 13'
+ '2013-02-10'
+ "2013-02"
+ "2013"
--- /dev/null
+Access
+------
+SiriDB knows the following access rights:
+
+* select
+* show
+* list
+* count
+* create
+* insert
+* drop
+* alter
+* grant
+* revoke
+
+The most obvious ones are combined into access profiles which can be used to grant
+or revoke multiple access rights at once.
+
+* read: (select, show, list and count)
+* write: *read* + (create and insert)
+* modify: *write* + (drop and alter)
+* full: *modify* + (grant, revoke)
+
+>**Warning**
+>
+>Changes to access rights are active immediately, so be careful when revoking
+>access rights from users.
--- /dev/null
+alter
+=====
+
+Syntax
+
+ alter <option> ...
+
+See available options for more information on each list command:
+
+- `alter database`: see `help alter database` for more information.
+- `alter group`: see `help alter group` for more information.
+- `alter server`: see `help alter server` for more information.
+- `alter servers`: see `help alter servers` for more information.
+- `alter user`: see `help alter user` for more information.
--- /dev/null
+alter database
+==============
+
+Syntax:
+
+ alter database set <option>
+
+Valid options are *drop_threshold*, *timezone*, *select_points_limit* and *list_limit*.
+
+drop_threshold
+--------------
+This value is used to protect you from accidentally dropping data from SiriDB.
+The threshold is a value between 0 and 1 (0/100%). The threshold value is only
+checked against the pool receiving your query. The default threshold value is
+1 (100%) but it might be a good idea to change this to a lower value.
+
+>**Note**
+>
+>Currently the drop_threshold is only used for dropping series and shards
+>because these are the only queries where we allow to drop multiple
+>entries at once.
+
+Example:
+
+ # Do not allow dropping more than 10% series or shards at once
+ alter database set drop_threshold 0.1
+
+ # View the current threshold
+ show drop_threshold
+
+set timezone
+------------
+Change the timezone for the database. When using a date/time in a query SiriDB
+needs to convert the given date to a timestamp. Default **NAIVE** is used which
+means SiriDB is naive about the time zone and acts as if it's a local time.
+
+>**Warning**
+>
+>When using a SiriDB database over multiple time zones it's probably best to
+>set the time zone to anything other than *NAIVE* since with *NAIVE* the server
+>*receiving* the query will convert the date to a local time-stamp. This means that
+>sending the same query to a server in another time zone could respond with
+>a different result.
+>
+>However, it's always possible in the query to specify
+>a UTC date by adding 'Z' to the date. For example: '2016-01-11 16:00Z' will
+>use UTC as it's time zone, no matter what time zone the database has configured.
+
+For a list of valid time zones see `help timezones`
+
+Example:
+
+ # Set the default time zone to UTC
+ alter database set timezone 'UTC'
+
+ # Set the default time zone to NAIVE
+ alter database set timezone 'NAIVE'
+
+ # Set the default time zone to Europe/Amsterdam
+ alter database set timezone 'Europe/Amsterdam'
+
+select\_points_limit
+-------------------
+Change the maximum points which can be returned by a select query. The default and recommended value is set to one million points. This value is chosen to prevent a single query for taking to much memory and ensures SiriDB can respond to almost any query in a reasonable amount of time.
+
+Example:
+
+ # Increase the select points limit to 5 million
+ alter database set select_points_limit 5000000
+
+list_limit
+----------
+Change the maximum value which can be used as a limit for a list statement. The default and recommended value is set to ten thousand to prevent queries which could take a large amount of memory. The value must be greater than or equal to 1000.
+
+Example:
+
+ # Set the list limit to 50 thousand.
+ alter database set list_limit 50000
+
\ No newline at end of file
--- /dev/null
+alter group
+===========
+
+Syntax:
+
+ alter group `groupname` set <option>
+
+Valid options are *expression* and *name*.
+
+set expression
+--------------
+Change the regular expression for a group.
+
+Example:
+
+ # Create group `linux`
+ create group `linux` for /linux.*/
+
+ # Change expression so we will match case insensitive
+ alter group `linux` set expression /linux.*/i
+
+set name
+--------
+Change the name for a group.
+
+>**Note**
+>
+>This statement expects a normal string using single or double quotes.
+>The reason is that 'set name' expects a string and not a group.
+
+Example:
+
+ # Rename group `linux` to `ubuntu`
+ alter group `linux` set name 'ubuntu'
+
\ No newline at end of file
--- /dev/null
+alter server
+============
+
+
+Syntax:
+
+ alter server <server_uuid / server_name> set <option>
+
+Valid options are *address*, *port*, *backup_mode* and *log_level*. We can use
+both, a servers name or uuid to change a server. To view the current servers
+names and uuids use the command: `list servers name, uuid`
+
+set address/port
+----------------
+Usually its not required to change the servers address or port using this
+command but instead you should change the address/port in the configuration
+file (default /etc/siridb/siridb.conf). When the server gets online it will
+contact all SiriDB servers and they will automatically update to the new
+address/port in their local database. However, if all servers in a cluster are
+updated at once, we need to tell at least one SiriDB server where to find the
+other server(s). This should be the only situation when this command is
+required.
+
+Example:
+
+ # srv1 and srv2 both have changed to another address so
+ # they are not able find each other. The command below
+ # is executed on srv1 and tells where to find srv2.
+
+ alter server 'srv2.old.domain:9010' set address 'srv2.new.domain'
+
+ # After executing the above command, srv1 will connect to srv2
+ # using the new domain name and announces its own new address so
+ # srv2 will update the address automatically and will connect
+ # to srv1 again.
+
+backup_mode
+-----------
+When a backup_mode is enabled on a SiriDB server, all files in the database
+directory will be closed (both dbpath and buffer_path). This way you can make
+a backup of SiriDB without having problems with open files.
+
+
+log_level
+--------
+With the argument *--log-level* it's possible to start with a certain log level.
+The default log level is *info*. If you want the log level to change while
+being online, this command can be used. It will *not* be saved when the server is
+restarted.
+
+Valid loglevels are "debug", "info", "warning", "error" and "critical"
+
+Example:
+
+ # Change the log-level to "debug"
+ alter server f851c6a4-820e-11e5-9661-080027f37001 set log_level debug
--- /dev/null
+alter server
+============
+
+
+Syntax:
+
+ alter servers [where...] set log_level <option>
+
+Valid options are *debug*, *info*, *warning*, *error* and *critical*.
+
+This command will change the log level for *n* servers at once. Changing the
+log level is explained in more detail at `help alter server`
--- /dev/null
+alter user
+==========
+
+Syntax:
+
+ alter user 'username' set <option>
+
+Valid options are *password* and *name*.
+
+Change a user name or password.
+
+Example:
+
+ # Change the password for "iris" to "siri"
+ alter user 'iris' set password 'siri'
--- /dev/null
+count
+=====
+
+Syntax:
+
+ count <option> ...
+
+See available options for more information on each *count* command:
+
+- `count groups`: see `help count groups` for more information.
+- `count pools`: see `help count pools` for more information.
+- `count series`: see `help count series` for more information.
+- `count servers`: see `help count servers` for more information.
+- `count shards`: see `help count shards` for more information.
+- `count users`: see `help count users` for more information.
--- /dev/null
+count groups
+============
+
+Syntax:
+
+ count groups [where ...]
+
+Count groups returns the number of groups defined in the database.
+
+Example:
+
+ # Get number of groups
+ count groups
+
+ # Get number of groups with more than 100 series
+ count groups where series > 100
+
+Example output:
+
+ {"groups": 23}
\ No newline at end of file
--- /dev/null
+count pools
+===========
+
+Syntax:
+
+ count pools [where ...]
+
+Count pools returns the number of pools in the SiriDB cluster.
+
+Examples:
+
+ # Get number of pools
+ count pools
+
+ # Get number of pools with less or equal to 100000 series
+ count pools where series <= 100000
+
+Example output:
+
+ {"pools": 2}
\ No newline at end of file
--- /dev/null
+count series
+============
+
+Syntax:
+
+ count series [length] [match_series] [where ...]
+
+Count series in the SiriDB cluster. For more information about how to select
+series see `help list series`
+
+Count series length gives the total number of points for the selected series.
+
+Examples:
+
+ # Get number of series
+ count series
+
+ # Get the total number of points in the database
+ count series length
+
+ # Get number of series in group `group_server01`
+ count series `group_server01`
+
+ # Get number of points for series in pool 0
+ count series length where pool == 0
+
+ # Get the number of series which started in the last week
+ count series where start > now - 1w and start <= now
+
+
+Example output (series):
+
+ {"series": 1105946}
+
+
+Example output (series length):
+
+ {"series_length": 77450345251}
\ No newline at end of file
--- /dev/null
+count servers
+=============
+
+Syntax:
+
+ count servers [received_points/selected_points] [where ...]
+
+Count servers returns the number of servers in a SiriDB cluster.
+Received points are the number of points received by a server since uptime.
+After a restart the received points counters are reset to zero.
+
+
+>**Info**
+>
+>Received points only shows the number of points after uptime. For the total
+>number of points you can use `count series length`
+
+Examples:
+
+ # Get number of servers
+ count servers
+
+ # Get number of servers in pool 0
+ count servers where pool == 0
+
+ # Get total number of received points since uptime
+ count servers received_points
+
+ # Get total number of selected (queried) points since uptime
+ count servers selected_points
+
+Example output (count servers):
+
+ {"servers": 6}
+
+Example output (count servers received_points):
+
+ {"servers_received_points": 21573435683}
--- /dev/null
+count shards
+============
+
+Syntax:
+
+ count shards [size] [where ...]
+
+Count shards returns the number of shards on all *online* servers in a SiriDB
+cluster. This means that offline servers are ignored and replica servers are
+included in the query.
+It's also possible to count the shards size in case you want to see the amount
+of disk-space shards are using.
+
+Example:
+
+ # Get number of shards
+ count shards
+
+ # Get number of shards for the current points. (assuming you have
+ # no shards for points in the future)
+ count shards where end > now
+
+ # Get the amount of disk space (in bytes) which shards are using
+ # on server01.
+ count shards size where server == 'server01'
+
+Example output (count shards):
+
+ {"shards": 51}
+
+Example output (count shards size):
+
+ {"shards_size": 355243846}
--- /dev/null
+count users
+===========
+
+Syntax:
+
+ count users [where ...]
+
+Count users returns the number of users.
+
+Examples:
+
+ # Get number of users
+ count users
+
+ # Get number of users not equal to 'iris'
+ count users where name != 'iris'
+
+Example output:
+
+ {"users": 6}
\ No newline at end of file
--- /dev/null
+create
+======
+
+Syntax
+
+ create <option> ...
+
+See available options for more information on each *count* command:
+
+- `create group`: see `help create group` for more information.
+- `create user`: see `help create user` for more information.
--- /dev/null
+create group
+============
+
+Syntax:
+
+ create group `groupname` for /regular_expression/
+
+Groups should be between **backticks** to make them different from strings.
+Since a group is basically a cached regular expression we need to provide
+the regular expression we want to use for the group.
+
+If you want to drop an existing group see `help drop group`.
+
+Example:
+
+ # Create a group linux
+ create group `linux` for /linux.*/
--- /dev/null
+create user
+===========
+
+Syntax:
+
+ create user 'my-username' set password 'my-password'
+
+Create a new user. This will create a new user without access to SiriDB.
+For more information on how to grant access to a user see `help grant`.
+
+Example:
+
+ # This will create a new user 'iris' with password 'siri'
+ create user 'iris' set password 'siri'
+
+ # Grant "read" access to "iris"
+ grant read to user "iris"
+
\ No newline at end of file
--- /dev/null
+drop
+====
+
+Syntax
+
+ drop <option> ...
+
+See available options for more information on each *drop* command:
+
+- `drop group`: see `help drop group` for more information.
+- `drop series`: see `help drop series` for more information.
+- `drop server`: see `help drop server` for more information.
+- `drop shards`: see `help drop shards` for more information.
+- `drop user`: see `help drop user` for more information.
--- /dev/null
+drop group
+==========
+
+Syntax:
+
+ drop group `groupname`
+
+Drops an existing group.
+
+Example:
+
+ # Drop group `linux`
+ drop group `linux`
\ No newline at end of file
--- /dev/null
+drop series
+===========
+
+Syntax:
+
+ drop series [series_match] [where ...] [set ignore_threshold true/false]
+
+Drops series from SiriDB. Optionally you can use a match and/or where statement
+to filter the series you want to drop. For more information about how to match
+series look at `help list series`.
+
+SiriDB has a mechanism to protect you from accidentally dropping all (or many)
+series. This is done with a *threshold* value. If the server receiving your drop
+request finds more series to drop than the threshold, the request
+is denied and you receive an *error_msg* about trying to delete more series than
+the threshold value. The *drop_threshold* value will not be checked by other
+servers in the cluster. You can view the current *drop_threshold* with `show drop_threshold` or look up `help alter database` for how to change this value. If you want to ignore the *drop_threshold* for one request you can
+add *set ignore_threshold true*. The default drop threshold is set to 1 (100%)
+which means you cannot drop *all* series but any other amount will pass. Any
+value between 0 and 1 will work. For example a value of 0.5 means you cannot
+drop more than 50% of the available series.
+
+>**Warning**
+>
+>Before dropping series using a regular expression you should check the
+>expression using `count series` and/or `list series` and see if your
+>match has the expected result.
+
+Examples:
+
+ # Drop series "series-001"
+ drop series "series-001"
+
+ # Drop all series
+ drop series set ignore_threshold true
--- /dev/null
+drop server
+===========
+
+Syntax:
+
+ drop server <server_uuid / server_name>
+
+Can be used to remove a server. We only allow dropping a server which has a
+replica since scaling down in number of pools is currently not supported.
+A server needs to be turned off before it can be dropped.
+
+>**Note**
+>
+>When having two servers in a pool, let's call them **siri1** and **siri2** and for some
+>reason **siri2** is broken and does not start. You might be in a situation where
+>**siri1** is waiting for **siri2** to connect and start to synchronize data. Both
+>servers are not working in this case but when dropping **siri2**, **siri1** removes
+>the *'wait for synchronization'* status and starts accepting inserts and queries.
+
+Example:
+
+ # Drop server 'siri2:9010'. We first need to turn off
+ # this server and make sure the server has a replica.
+ drop server 'siri2:9010'
--- /dev/null
+drop shards
+===========
+
+Syntax:
+
+ drop shards [where ...] [set ignore_threshold true/false]
+
+Drops an existing shard using the shard id (sid). Use `list shards` for an
+overview of the current shards. This statement requires all pools to have at
+least one online server. The number of dropped shard in the result message
+only contains the dropped shards by one member of a pool, not the shards which
+are dropped by a replica. This is different from the `count shards` statement
+which includes shards on replica servers as well.
+
+>**Note**
+>
+>This statement is protected with a *threshold*. See `help drop series` and
+>`help alter database` for more information about this threshold value.
+
+Example:
+
+ # Drop shards for points which are older than one year
+ drop shards where end < now - 52w
--- /dev/null
+drop user
+=========
+
+Syntax:
+
+ drop user `username`
+
+Drops an existing user.
+
+>**Warning**
+>
+>If accidentally all users with access are dropped, you need to recover the
+>default user. See `help noaccess` on how to recover from a situation in which you
+>don't have access to SiriDB.
+
+Example:
+
+ # Drop the default user "siri"
+ drop user "siri"
--- /dev/null
+Aggregate functions
+===================
+SiriDB supports multiple build-in aggregation and filter functions. Using these functions can be useful to reduce network traffic. Note that multiple functions can be combined using the arrow `=>` sign. (see `help select` for more information on how to use and combine functions)
+
+Most aggregation function accept an optional `ts` argument. When not providing the `ts` argument, SiriDB will usually return the last timestamp in the result. One exception is the `first()` function which will return the first timestamp instead.
+
+For example:
+
+ # Select the last time-stamp and the average over all values.
+ select mean() from `my_series`
+
+ # Select the first time-stamp and first value:
+ select first() from `my_series`
+
+List of supported aggregation and filter functions:
+
+limit
+-----
+Syntax:
+
+ limit(max_points, aggr_function)
+
+Returns at most `max_points` and uses a given aggregation function to reduce
+the number of points if needed.
+
+Example:
+
+ # Returns at most 100 points for 'my-series'. The original values are
+ # returned in case hundred or less points are found. In case more points
+ # are found a mean aggregation function is used.
+ select limit(100, mean) from "my-series"
+
+count
+-----
+Syntax:
+
+ count([ts])
+
+Returns an integer value.
+
+Count can be used to calculate points over a period of time.
+
+Example:
+
+ # Get the number of points in 'series-001' over the past 24 hours.
+ select count(now) from "series-001" between now - 1d and now
+
+sum
+---
+Syntax:
+
+ sum([ts])
+
+Returns an integer or float value depending on the series data type.
+
+Sum can be used when you want to know the sum of the values over a period of time.
+
+Example:
+
+ # Get the sum of the values collected over the last 24 hours per hour.
+ select sum(1h) from "series-001" between now - 1d and now
+
+
+max
+---
+Syntax:
+
+ max([ts])
+
+Returns an integer or float value depending on the series data type.
+
+Max can be used to identify the highest value in the selected time window.
+
+Example:
+
+ # Get the maximum value in 'series-001' over the last week.
+ select max(now) from "series-001" between now - 1w and now
+
+
+min
+---
+Syntax:
+
+ min([ts])
+
+Returns an integer or float value depending on the series data type.
+
+Min is the opposite of max, you identify the lowest value in the selected time window.
+
+Example:
+
+ # Get the minimum value per day from 'series-001' between two dates.
+ select min(1d) from "series-001" between '2016-11-14' and '2016-11-21'
+
+
+mean
+----
+Syntax:
+
+ mean([ts])
+
+Returns a float value.
+
+Mean is used to calculate the average values per selected time window.
+
+Example:
+
+ # Get average value of 'series-001' up until now.
+ select mean(now) from "series-001" before now
+
+
+median
+------
+Syntax:
+
+ median([ts])
+
+Returns a float value.
+
+The median is a robust measure of central location, and is less affected by the presence of outliers in your data. When the number of data points is odd, the middle data point is returned as float value. When the number of data points is even, the median is interpolated by taking the average of the two middle values.
+
+median_high
+-----------
+Syntax:
+
+ median_high([ts])
+
+Returns an integer or float value depending on the series data type.
+
+The high median is always a member of the data set. When the number of data points is odd, the middle value is returned. When it is even, the larger of the two middle values is returned.
+
+median\_low
+-----------
+Syntax:
+
+ median_low([ts])
+
+Returns an integer or float value depending on the series data type.
+
+The low median is always a member of the data set. When the number of data points is odd, the middle value is returned. When it is even, the smaller of the two middle values is returned.
+
+variance
+--------
+Syntax:
+
+ variance([ts])
+
+Returns a float value.
+
+Returns the sample variance of data, an iterable of at least two real-valued numbers. Variance, or second moment about the mean, is a measure of the variability (spread or dispersion) of data. A large variance indicates that the data is spread out; a small variance indicates it is clustered closely around the mean.
+
+pvariance
+---------
+Syntax:
+
+ pvariance([ts])
+
+Returns a float value.
+
+Returns the population variance of data, a non-empty iterable of real-valued numbers. Variance, or second moment about the mean, is a measure of the variability (spread or dispersion) of data. A large variance indicates that the data is spread out; a small variance indicates it is clustered closely around the mean.
+
+stddev
+------
+Syntax:
+
+ stddev([ts])
+
+Returns a float value.
+
+Returns the standard deviation which is the square root of its variance.
+
+difference
+----------
+Syntax:
+
+ difference([ts])
+
+Returns an integer or float value depending on the series data type.
+
+Difference without arguments is used to get the difference between values.
+As an optional argument you can specify a time period. In this case the function returns the difference between the first value and the last value within the time window.
+
+Example:
+
+ # Select difference between values in series-001.
+ select difference() from 'series-001'
+
+derivative
+----------
+Syntax:
+
+ derivative([ts [, ts]])
+
+Returns a float value.
+
+The derivative can be used to get the difference per time unit. When no optional arguments
+are used we return the difference per one unit. For example, in a database with second
+precision the return value will be the difference per second. Optionally another time unit can
+be used. A second argument can be used to set a time period. This time period will be used to get the difference between the first and last value within the time window.
+
+Example:
+
+ # Select the difference per second for values in series-001.
+ select derivative(1s) from 'series-001'
+
+ # Select the difference per second between the first and last value
+ # within each hour for values in 'series-001'
+ select derivative(1s, 1h) from 'series-001'
+
+
+filter
+------
+Syntax:
+
+ filter(<operator> <value>)
+
+Returns an integer, float or string value depending on the series data type.
+
+Filter is used to filter the result by values.
+
+Example:
+
+ # Select all values from 'series-001' except where the value is 0
+ select filter(!= 0) from 'series-001'
+
+ # Select all positive values from 'series-001'
+ select filter(> 0) from 'series-001'
+
+ # Select all values containing 'error' and not 'unavailable
+ select filter(~'error') => filter(!~'unavailable') from 'some-log-series'
+
+ # Select all values starting with 'error' using a regular expression
+ select filter(/error.*/) from 'some-log-series'
+
+ # Filter out Not-a-number (nan) and Infinite values
+ select filter(!=nan) => filter(!=inf) => filter(!=-inf) from 'some-float-series'
+
+
+first
+-----
+Syntax:
+
+ first([ts])
+
+Returns the first value in each `ts` time window. (Or just the first value)
+
+Example:
+
+ # Select the first value from 'series-001'
+ select first() from 'series-001'
+
+ # Select the first value per day from 'series-001'
+ select first(1d) from 'series-001'
+
+ # Select the first value in 2018 from 'series-001'
+ select first() from 'series-001' after '2018'
+
+
+last
+----
+Syntax:
+
+ last([ts])
+
+Returns the last value in each `ts` time window. (Or just the last value)
+
+Example:
+
+ # Select the last value from 'series-001'
+ select last() from 'series-001'
+
+ # Select the last value per day from 'series-001'
+ select last(1d) from 'series-001'
+
+ # Select the last value in 2017 from 'series-001'
+ select last() from 'series-001' before '2018'
\ No newline at end of file
--- /dev/null
+grant
+=====
+
+Syntax:
+
+ grant <access> to user 'username'
+
+Grants access rights to a user. For information about access rights
+see `help access`.
+
+Example:
+
+ # Grant drop and create to user "iris"
+ grant drop, create to user "iris"
+
+
+Output:
+
+ {"success_msg": "Successfully granted permissions to user ..."}
+
\ No newline at end of file
--- /dev/null
+list
+====
+
+Syntax
+
+ list <option> ...
+
+See available options for more information on each *list* command:
+
+- `list groups`: see `help list groups` for more information.
+- `list pools`: see `help list pools` for more information.
+- `list series`: see `help list series` for more information.
+- `list servers`: see `help list servers` for more information.
+- `list shards`: see `help list shards` for more information.
+- `list users`: see `help list users` for more information.
+
+
+>**Warning**
+>
+>Each list command allows a limit option which is set to 1000 by default but
+>can be any integer value from 1 to 10000. Usually we only expect to reach this
+>limit with listing series or shards, but note that it's always there. We have
+>set the 1000 limit by default to prevent accidentally listing all series
+>(its possible to have millions of series).
--- /dev/null
+list groups
+===========
+
+syntax
+
+ list groups [columns] [where ...] [limit ...]
+
+columns
+-------
+Valid columns are:
+
+- name: Group name
+- series: Number of series in the group.
+- expression: Show the expression used for this group.
+
+When no columns are provided the default is used. (name)
+
+Examples:
+
+ # View all groups
+ list groups
+
+ # View groups and the expression used
+ list groups name, expression
+
+ # sample output (list groups)
+ {
+ "columns": ["name"],
+ "groups": [
+ ["linux"],
+ ["windows"]
+ ]
+ }
+
\ No newline at end of file
--- /dev/null
+list pools
+==========
+
+syntax
+
+ list pools [columns] [where ...] [limit ...]
+
+columns
+-------
+Valid columns are:
+
+- pool: Pool ID
+- servers: Number of servers in the pool.
+- series: Number of series in the pool.
+
+When no columns are provided the default is used. (pool, servers, series)
+
+Example:
+
+ # View pools
+ list pools
+
\ No newline at end of file
--- /dev/null
+list series
+===========
+
+syntax
+
+ list series [columns] [match_series] [where ...] [limit ...]
+
+columns
+-------
+Valid columns are:
+
+- name: Series name
+- pool: Pool where the series is assigned to
+- start: Time-stamp of the first value in the series
+- end: Time-stamp of the last value in the series
+- length: The number of points in a series
+- type: The series type. ("integer" or "float")
+
+When no columns are provided the default is used. (name)
+
+match series
+------------
+
+Series can be matched using different methods. Groups can help to quickly get the required series even in a database with millions of unique series.
+
+
+syntax:
+
+ <*/all | series_name | regular_expression | group> [update_function <match_series>]
+
+*/all
+-----
+All series are selected.
+
+series name
+-----------
+A series name is a string containing the series name. An error is raised when the given series name is not found in SiriDB. (Note that this exception is not raised when the database is re-indexing. In case of re-indexing we just don't return the series like a search using regular expression)
+
+An advantage of using series names in a SiriDB cluster is that we know in which pool the series exists. The given query will therefore only be send to the applicable pool(s). We don't know which pool has series when using a regular expression or group match so each pool then needs the query.
+
+Example:
+
+ list series 'series-001'
+
+regular expression
+------------------
+Regular expressions can be used to select series. Note that each pool in a SiriDB cluster will look for matching series. If you plan to use a regular expression multiple times, you should consider creating a group for the expression.
+
+Example:
+
+ # list all series starting with "linux"
+ list series /linux.*/
+
+ # list all series starting with "linux" (case-insensitive)
+ list series /linux.*/i
+
+ # list all series not starting with "linux"
+ list series /(?!linux).*/
+
+ # list all series not containing "linux"
+ list series /((?!linux).)*/
+
+group
+-----
+Groups are basically cached regular expression and can be used together with normal
+regular expressions. When you use a regular expression to match series in a group it's
+best to first select the group and then the regular expression. This way the regular
+expression only needs to validate series inside the group.
+
+Examples:
+
+ # list all series in group "linux"
+ list series `linux`
+
+ # list all series in group "linux" with "cpu" in the name
+ # note that we first select the group so the regular expression only
+ # needs to be validated on series in the group.
+ list series `linux` & /.*cpu.*/
+
+update functions
+----------------
+When selecting series you can combine *series-names*, *regular-expressions* and *groups*. Update functions tell SiriDB how to combine selection.
+SiriDB knows four update functions:
+
+* difference (alias: **-**)
+* symmentric_difference (alias: **^**)
+* union (aliases: **,** and **|**)
+* intersection (alias: **&**)
+
+examples
+--------
+
+ # list multiple series using union (we actually use the alias here)
+ list series 'series-001', 'series-002', 'series-003'
+
+ # list series in group "linux" except series which are also in group "cpu"
+ list series `linux` - `cpu`
+
+ # list series when member of group "linux" or group "cpu" but not both
+ list series `linux` ^ `cpu`
+
+ # list series that are both members of `linux` and `cpu` except when
+ # a series name contains "test".
+ list series `linux` & `cpu` - /.*test.*/
+
+ # list series in group `linux` and view their length.
+ list series name, length `linux`
+
+ # list series in group `linux` which have their last data point more
+ # than 100 days ago
+ list series `linux` where end < now - 100d
+
+
+ # sample output (list series)
+ {
+ "columns": ["name"],
+ "series": [
+ ["series-001"],
+ ["series-002"]
+ ]
+ }
--- /dev/null
+list servers
+============
+
+syntax
+
+ list servers [columns] [where ...] [limit ...]
+
+List servers in a SiriDB Cluster. This command can be useful to view status
+information about a server.
+
+columns
+-------
+Valid columns are:
+
+- active_handles: Returns the active handles which can be used as an indicator on how busy a server is.
+- active_tasks: Returns the active tasks for the current database.
+- address: Server address.
+- buffer_path: Path where this server keeps the buffer file.
+- buffer_size: Size the server uses for one series in the buffer.
+- dbpath: Path where the server stores the database.
+- fifo_files: Number of fifo files which are used to update the replica server. This value is 0 if the server has no replica. A value greater than 1 could be an indication that replication is not working.
+- idle_percentage: Returns percentage of idle time since the database was loaded.
+- idle_time: Returns the idle time in seconds since the database was loaded.
+- ip_support: IP Support setting on the server. (ALL/ IPV4ONLY/ IPV6ONLY)
+- libuv: Version of libuv library.
+- log_level: Current loglevel for the server.
+- max\_open\_files: Returns the maximum open files value used for sharding on *this* server. (If this value is lower than expected, please check the log files for SiriDB as startup time)
+- mem_usage: Shows memory usage for the server in MB's.
+- name: Server name.
+- online: True when the server is online.
+- open_files: Number of open files for *this* database on the server.
+- pool: Returns the pool ID for the server.
+- port: Server port.
+- received_points: Returns the number of received points by the server. On each restart of the SiriDB Server the counter will reset to 0. This value is only incremented when the server is receiving points from a client.
+- reindex_progress: Returns the re-index status. Only available when the database is re-indexing series over pools.
+- selected_points: Returns the selected points on the server. On each restart of the SiriDB Server the counter will reset to 0. This value includes all points which are read from the local shards and the points received from other servers to respond to a select query. The value is only incremented when the server received the select query from a client.
+- startup_time: Time it takes to start the server.
+- status: Current server status.
+- sync_progress: Return synchronization status while creating a new replica server.
+- uptime: Uptime in seconds.
+- uuid: Server UUID (unique ID)
+- version: SiriDB version
+
+When no columns are provided the default is used. (name, pool, version, online, status)
+
+examples
+--------
+
+ # list all servers in a SiriDB cluster.
+ list servers
+
+ # list all offline servers
+ list servers where online == false
+
+ # view memory usage and open files on all servers.
+ list servers name, mem_usage, open_files
+
+
+ # sample output (list servers)
+ {
+ "columns": ["name", "pool", "version", "online", "status"],
+ "servers": [
+ ["siri1:9010", 0, "2.0.10", true, "running"],
+ ["siri2:9010", 1, "2.0.10", true, "running"]
+ ]
+ }
--- /dev/null
+list shards
+===========
+
+syntax
+
+ list shards [columns] [where ...] [limit ...]
+
+columns
+-------
+Valid columns are:
+
+- start: Start timestamp for the shard
+- end: End timestamp for the shard
+- sid: Shard identifier (the same sid usually exist on multiple servers).
+- server: Server name on which the shard exists.
+- pool: Pool where the shard in exists.
+- status: Status flags for the shard.
+- type: Type of the shard (number or log).
+- size: Size of the shard. This is the total shard size over all pools. When a
+ pool has more servers (replicas) the displayed size can vary when running
+ this query multiple times because servers are responsible for optimizing
+ their own shards and this could result in different shard sizes.
+
+When no columns are provided the default is used. (sid, pool, server, start, end)
+
+Example:
+
+ # List all shards
+ list shards
+
+ # List shards used for data older then 100 days
+ list shards where start < now - 100d
+
+ # sample output (list shards)
+ {
+ "columns": ["sid", "pool", "server", "start", "end"
+ ],
+ "shards": [
+ [1449705600, 0, "srv01:9010", 1449705600, 1450310400],
+ [1449705601, 0, "srv01:9010", 1449705600, 1450310400],
+ ...
+ ]
+ }
--- /dev/null
+list users
+==========
+
+syntax
+
+ list users [columns] [where ...] [limit ...]
+
+columns
+-------
+Valid columns are:
+
+- name: User name.
+- access: Access rights assigned to the user.
+
+When no columns are provided the default is used. (name, access)
+
+Example:
+
+ # List all users
+ list users
+
+ # List users with full access
+ list users where access == full
+
\ No newline at end of file
--- /dev/null
+Lost access to SiriDB
+---------------------
+Read/use this section when accidentally all access rights to a database are
+gone. Follow the steps below to recover the default user.
+(username: *iris*, password: *siri*)
+
+Stop **all servers** in the SiriDB cluster.
+
+ # Assume you use systemd to start SiriDB...
+ > sudo systemctl stop siridb-server.service
+
+>**Warning**
+>
+>It's really important to stop all servers in a SiriDB cluster to prevent
+>database inconsistency.
+
+Remove the appropriate file on **all servers** in the SiriDB cluster.
+
+ # Restore the default user
+ > rm my_database_folder/users.dat
+
+Start **all servers** in the SiriDB cluster.
+
+ # Assume systemd is used to start SiriDB...
+ > sudo systemctl start siridb-server.service
--- /dev/null
+revoke
+======
+
+Syntax:
+
+ revoke <access> from 'username'
+
+Revokes access rights from a user. For information about access rights
+see `help access`.
+
+>**Warning**
+>
+>If accidentally all access rights for all users are gone, you need to recover
+>the default user. See `help noaccess` for how to recover from a situation
+>not having access to SiriDB.
+
+Example:
+
+ # Revoke drop and create from user "iris"
+ revoke drop, create from user "iris"
+
+
+Output:
+
+ {"success_msg": "Successfully revoked permissions from ..."}
\ No newline at end of file
--- /dev/null
+select
+======
+
+Syntax:
+
+ select <points/functions> from <match_series [<where>]> [<time_range>] [<merge_data>]
+
+Example:
+
+ # Select points from "series-001"
+ select * from "series-001"
+
+functions
+---------
+It's possible to select multiple aggregate functions in one query. This has some
+advantages over performing multiple queries since the database in this case only
+needs to search for the series and points once. To find the requested aggregate
+in the result we must add a prefix and/or suffix to the series name to make the
+name unique. Note that a prefix and/or suffix is only required when querying
+multiple aggregates.
+
+Example:
+
+ # Select both the min and max grouped by 5 minutes from "series-001"
+ select min(5m) prefix "min-", max (5m) prefix "max-" from "series-001"
+
+For more help on aggregate functions see `help functions`.
+
+Combine functions
+-----------------
+The aggregate functions can be used together by parsing the result of one function
+to the next. It's also possible to use the same function twice which can be
+useful with for example difference.
+
+Example:
+
+ # Select the median grouped by 1 minute and return the difference for that result
+ select median (1m) => difference () from "series-001"
+
+match_series
+------------
+see `help list series` on how to match series.
+
+time_range
+----------
+An optional time range can be given to select only a part of the series data.
+If no time range is provided SiriDB returns all the data available. As a time
+range we can use *before*, *after* or *between .. and ..*
+
+When using *after* you basically set a *start* time, with *before* a *end* time
+and when using *between .. and ..* you set both a *start* and *end* time.
+Points having the exact the start time are *included* in the result, points
+with the exact end time are *excluded* from the result.
+
+>**Note**
+>
+>It's safe to use *now* multiple times in query. SiriDB only calculates *now* one
+>time while processing a query. This way you can be sure that *now* has the
+>same value.
+
+Examples:
+
+ # Select all points from "series-001" in the last 24h
+ select * from "series-001" after now - 1d
+
+ # Select all points from "series-001" today
+ select * from "series-001" between now - (now % 1d) and now - (now % 1d) + 1d
+
+ # Select all points from "series-001" before November, 2015
+ select * from "series-001" before "2015-11"
+
+merge_data
+----------
+When selecting points from multiple series you can merge the data together in
+result. Most of the time you also want to provide an aggregate function with the
+merge so series get really merged into one series. Even with merge it's still
+possible to use aggregate functions on the series before they are merged.
+
+>**Note**
+>
+>Sometimes it does not matter for the end result if you use an aggregate
+>function on the series or only while merging. However, if you have multiple
+>pools it can be an advantage to aggregate the series and the merge. This is
+>an advantage because each pool can do some aggregate work and only send the
+>aggregated result to the server processing the query.
+>
+>For example:
+>
+>`select * from /series.*/ merge as "series" using sum(1h)`
+>
+>will have the exact same result as
+>
+>`select sum(1h) from /series.*/ merge as "series" using sum(1h)`
+>
+>but the last one will be faster, assuming you are using a SiriDB cluster and
+>/series.*/ contains multiple series spread out over multiple pools.
+
+Examples:
+
+>**Note**
+>
+>In the examples below we assume there are no points in the future. If you have
+>points in the future and only want points from 7 days ago up till now you can
+>use between now - 7d and now. Since we assume our series have no points in the
+>future we use after now - 7d.
+
+ # We want the average value per 1 hour over the last 7 days over s01
+ # and s02. The series should weight equal to each other but s01 has
+ # a point each 2 seconds while s02 only has a point each 5 seconds.
+ # We solve this by first getting the mean value for each series
+ # by 1 hour before merging the series.
+ #
+ # Note that we use mean(1) while merging. This means we group by 1 second or
+ # millisecond depending on the time precision. We can do this because the
+ # series are already grouped by 1h and therefore have re-indexed timestamps
+ # at precisely each hour.
+ select mean(1h) from "s01", "s02" after now - 7d merge as "merged_s" using mean(1)
+
+ # We want the number of points s01 and s02 have over the last 7 days.
+ # Note: when having no timestamps after now this will result in one
+ # value with timestamp *now*
+ select count(now) from "s01", "s02" after now - 7d merge as "merged_s" using sum(1)
+
+ # We have s01 and s02 representing counter data. We want to sum the
+ # values per 4 hours over January, 2015 and show this as one series.
+ select sum(4h) from "s01", "s01" between "2015-01" and "2015-02" merge as "merged_s" using sum(1)
--- /dev/null
+show
+====
+
+Syntax
+
+ show [<option> [,<option> [...]]
+
+See available options for more info on each show command:
+
+- `show active_handles`: Returns the active handles which can be used as an indicator for how busy a server is.
+- `show active_tasks`: Returns the active tasks for the current database.
+- `show buffer_path`: Returns the local buffer path on *this* server.
+- `show buffer_size`: Returns the buffer size in bytes on *this* server.
+- `show dbname`: Returns the database name.
+- `show dbpath`: Returns the local database path on *this* server.
+- `show drop_threshold`: Returns the current drop threshold (value between 0 and 1 representing a percentage).
+- `show duration_log`: Returns the sharding duration for log data on *this* database (not supported yet).
+- `show duration_num`: Returns the sharding duration for num data on *this* database.
+- `show fifo_files`: Returns the number of fifo files which are used to update the replica server. This value is 0 if the server has no replica. A value greater than 1 could be an indication that replication is not working.
+- `show idle_percentage`: Returns percentage of idle time since the database was loaded.
+- `show idle_time`: Returns the idle time in seconds since the database was loaded.
+- `show ip_support`: Returns the ip support setting on *this* server.
+- `show libuv`: Returns the version of libuv on *this* server.
+- `show list_limit`: Returns the maximum value which can be used as limit in a list query.
+- `show log_level`: Returns the current log level for *this* server.
+- `show max_open_files`: Returns the maximum open files value used for sharding on *this* server (if this value is lower than expected, please check the log files for SiriDB as startup time).
+- `show mem_usage`: Returns the current memory usage in MB's on *this* server.
+- `show open_files`: Returns the number of open files on *this* server for the selected database (should be 0 when the server is in backup_mode).
+- `show pool`: Returns the pool ID for *this* server.
+- `show received_points`: Returns the number of received points for *this* server. On each restart of the SiriDB Server the counter will reset to 0. This value is only incremented when *this* server is receiving points from a client.
+- `show reindex_progress`: Returns the re-index status on *this* server. Only available when the database is re-indexing series over pools.
+- `show selected_points`: Returns the selected points for *this* server. On each restart of the SiriDB Server the counter will reset to 0. This value includes all points which are read from the local shards and the points received from other servers to respond to a select query. The value is only incremented when *this* server received the select query from a client.
+- `show select_points_limit`: Returns the maximum number of points which can be returned with a select query.
+- `show server`: Returns *this* server name. The name has format *host:port*
+- `show startup_time`: Returns the time in seconds it took to startup the SiriDB database on *this* server.
+- `show status`: Returns the current status for *this* server.
+- `show sync_progress`: Return synchronization status while creating a new replica server on *this* server.
+- `show time_precision`: Returns the time precision for *this* database.
+- `show timezone`: Returns the timezone for *this* database.
+- `show uptime`: Returns the uptime in seconds *this* server is running.
+- `show uuid`: Returns the UUID (unique ID) for *this* server.
+- `show version`: Returns the SiriDB version running on *this* server.
+- `show who_am_i`: Returns the user who is running *this* request.
+
+examples
+--------
+
+ # show the database name and time precision
+ show dbname, time_precision
+
+
+ # sample output
+ {
+ "data": [
+ {
+ "name": "dbname",
+ "value": "mydb"
+ },
+ {
+ "name": "time_precision",
+ "value": "s"
+ }
+ ]
+ }
--- /dev/null
+timeit
+======
+
+Can be placed in front of any query and will return information about the time it took to process the query.
+
+Syntax:
+
+ timeit <any_query>
+
+Example result:
+
+ {
+ "__timeit__": [
+ {
+ "time": 0.001156334212755393,
+ "server": "server04.siridb.net:9010"
+ },
+ {
+ "time": 0.001481771469116211,
+ "server": "server01.siridb.net:9010"
+ }
+ ]
+ }
+
+Here `__timeit__` is an array containing response data from each server involved in processing the query. The last server in this list is the server who has received the query. Since this server is responsible for sending the response it has to wait for all other servers to complete and therefore the query time for this server will always be the highest value of all servers in the list.
--- /dev/null
+List of TimeZones
+=================
+
+For information on how to change the time zone see `help alter database`
+
+>**Note**
+>
+>There is one special time-zone, named **NAIVE** which is the default time-zone
+>when creating a database. When this time-zone is used, a SiriDB server handles
+>a given date as a local time. All other time zones can be found below.
+
+- Africa/Abidjan
+- Africa/Accra
+- Africa/Addis\_Ababa
+- Africa/Algiers
+- Africa/Asmara
+- Africa/Bamako
+- Africa/Bangui
+- Africa/Banjul
+- Africa/Bissau
+- Africa/Blantyre
+- Africa/Brazzaville
+- Africa/Bujumbura
+- Africa/Cairo
+- Africa/Casablanca
+- Africa/Ceuta
+- Africa/Conakry
+- Africa/Dakar
+- Africa/Dar\_es\_Salaam
+- Africa/Djibouti
+- Africa/Douala
+- Africa/El\_Aaiun
+- Africa/Freetown
+- Africa/Gaborone
+- Africa/Harare
+- Africa/Johannesburg
+- Africa/Juba
+- Africa/Kampala
+- Africa/Khartoum
+- Africa/Kigali
+- Africa/Kinshasa
+- Africa/Lagos
+- Africa/Libreville
+- Africa/Lome
+- Africa/Luanda
+- Africa/Lubumbashi
+- Africa/Lusaka
+- Africa/Malabo
+- Africa/Maputo
+- Africa/Maseru
+- Africa/Mbabane
+- Africa/Mogadishu
+- Africa/Monrovia
+- Africa/Nairobi
+- Africa/Ndjamena
+- Africa/Niamey
+- Africa/Nouakchott
+- Africa/Ouagadougou
+- Africa/Porto-Novo
+- Africa/Sao\_Tome
+- Africa/Tripoli
+- Africa/Tunis
+- Africa/Windhoek
+- America/Adak
+- America/Anchorage
+- America/Anguilla
+- America/Antigua
+- America/Araguaina
+- America/Argentina/Buenos\_Aires
+- America/Argentina/Catamarca
+- America/Argentina/Cordoba
+- America/Argentina/Jujuy
+- America/Argentina/La\_Rioja
+- America/Argentina/Mendoza
+- America/Argentina/Rio\_Gallegos
+- America/Argentina/Salta
+- America/Argentina/San\_Juan
+- America/Argentina/San\_Luis
+- America/Argentina/Tucuman
+- America/Argentina/Ushuaia
+- America/Aruba
+- America/Asuncion
+- America/Atikokan
+- America/Bahia
+- America/Bahia\_Banderas
+- America/Barbados
+- America/Belem
+- America/Belize
+- America/Blanc-Sablon
+- America/Boa\_Vista
+- America/Bogota
+- America/Boise
+- America/Cambridge\_Bay
+- America/Campo\_Grande
+- America/Cancun
+- America/Caracas
+- America/Cayenne
+- America/Cayman
+- America/Chicago
+- America/Chihuahua
+- America/Costa\_Rica
+- America/Creston
+- America/Cuiaba
+- America/Curacao
+- America/Danmarkshavn
+- America/Dawson
+- America/Dawson\_Creek
+- America/Denver
+- America/Detroit
+- America/Dominica
+- America/Edmonton
+- America/Eirunepe
+- America/El\_Salvador
+- America/Fort\_Nelson
+- America/Fortaleza
+- America/Glace\_Bay
+- America/Godthab
+- America/Goose\_Bay
+- America/Grand\_Turk
+- America/Grenada
+- America/Guadeloupe
+- America/Guatemala
+- America/Guayaquil
+- America/Guyana
+- America/Halifax
+- America/Havana
+- America/Hermosillo
+- America/Indiana/Indianapolis
+- America/Indiana/Knox
+- America/Indiana/Marengo
+- America/Indiana/Petersburg
+- America/Indiana/Tell\_City
+- America/Indiana/Vevay
+- America/Indiana/Vincennes
+- America/Indiana/Winamac
+- America/Inuvik
+- America/Iqaluit
+- America/Jamaica
+- America/Juneau
+- America/Kentucky/Louisville
+- America/Kentucky/Monticello
+- America/Kralendijk
+- America/La\_Paz
+- America/Lima
+- America/Los\_Angeles
+- America/Lower\_Princes
+- America/Maceio
+- America/Managua
+- America/Manaus
+- America/Marigot
+- America/Martinique
+- America/Matamoros
+- America/Mazatlan
+- America/Menominee
+- America/Merida
+- America/Metlakatla
+- America/Mexico\_City
+- America/Miquelon
+- America/Moncton
+- America/Monterrey
+- America/Montevideo
+- America/Montserrat
+- America/Nassau
+- America/New\_York
+- America/Nipigon
+- America/Nome
+- America/Noronha
+- America/North\_Dakota/Beulah
+- America/North\_Dakota/Center
+- America/North\_Dakota/New\_Salem
+- America/Ojinaga
+- America/Panama
+- America/Pangnirtung
+- America/Paramaribo
+- America/Phoenix
+- America/Port-au-Prince
+- America/Port\_of\_Spain
+- America/Porto\_Velho
+- America/Puerto\_Rico
+- America/Rainy\_River
+- America/Rankin\_Inlet
+- America/Recife
+- America/Regina
+- America/Resolute
+- America/Rio\_Branco
+- America/Santa\_Isabel
+- America/Santarem
+- America/Santiago
+- America/Santo\_Domingo
+- America/Sao\_Paulo
+- America/Scoresbysund
+- America/Sitka
+- America/St\_Barthelemy
+- America/St\_Johns
+- America/St\_Kitts
+- America/St\_Lucia
+- America/St\_Thomas
+- America/St\_Vincent
+- America/Swift\_Current
+- America/Tegucigalpa
+- America/Thule
+- America/Thunder\_Bay
+- America/Tijuana
+- America/Toronto
+- America/Tortola
+- America/Vancouver
+- America/Whitehorse
+- America/Winnipeg
+- America/Yakutat
+- America/Yellowknife
+- Antarctica/Casey
+- Antarctica/Davis
+- Antarctica/DumontDUrville
+- Antarctica/Macquarie
+- Antarctica/Mawson
+- Antarctica/McMurdo
+- Antarctica/Palmer
+- Antarctica/Rothera
+- Antarctica/Syowa
+- Antarctica/Troll
+- Antarctica/Vostok
+- Arctic/Longyearbyen
+- Asia/Aden
+- Asia/Almaty
+- Asia/Amman
+- Asia/Anadyr
+- Asia/Aqtau
+- Asia/Aqtobe
+- Asia/Ashgabat
+- Asia/Baghdad
+- Asia/Bahrain
+- Asia/Baku
+- Asia/Bangkok
+- Asia/Beirut
+- Asia/Bishkek
+- Asia/Brunei
+- Asia/Chita
+- Asia/Choibalsan
+- Asia/Colombo
+- Asia/Damascus
+- Asia/Dhaka
+- Asia/Dili
+- Asia/Dubai
+- Asia/Dushanbe
+- Asia/Gaza
+- Asia/Hebron
+- Asia/Ho\_Chi\_Minh
+- Asia/Hong\_Kong
+- Asia/Hovd
+- Asia/Irkutsk
+- Asia/Jakarta
+- Asia/Jayapura
+- Asia/Jerusalem
+- Asia/Kabul
+- Asia/Kamchatka
+- Asia/Karachi
+- Asia/Kathmandu
+- Asia/Khandyga
+- Asia/Kolkata
+- Asia/Krasnoyarsk
+- Asia/Kuala\_Lumpur
+- Asia/Kuching
+- Asia/Kuwait
+- Asia/Macau
+- Asia/Magadan
+- Asia/Makassar
+- Asia/Manila
+- Asia/Muscat
+- Asia/Nicosia
+- Asia/Novokuznetsk
+- Asia/Novosibirsk
+- Asia/Omsk
+- Asia/Oral
+- Asia/Phnom\_Penh
+- Asia/Pontianak
+- Asia/Pyongyang
+- Asia/Qatar
+- Asia/Qyzylorda
+- Asia/Rangoon
+- Asia/Riyadh
+- Asia/Sakhalin
+- Asia/Samarkand
+- Asia/Seoul
+- Asia/Shanghai
+- Asia/Singapore
+- Asia/Srednekolymsk
+- Asia/Taipei
+- Asia/Tashkent
+- Asia/Tbilisi
+- Asia/Tehran
+- Asia/Thimphu
+- Asia/Tokyo
+- Asia/Ulaanbaatar
+- Asia/Urumqi
+- Asia/Ust-Nera
+- Asia/Vientiane
+- Asia/Vladivostok
+- Asia/Yakutsk
+- Asia/Yekaterinburg
+- Asia/Yerevan
+- Atlantic/Azores
+- Atlantic/Bermuda
+- Atlantic/Canary
+- Atlantic/Cape\_Verde
+- Atlantic/Faroe
+- Atlantic/Madeira
+- Atlantic/Reykjavik
+- Atlantic/South\_Georgia
+- Atlantic/St\_Helena
+- Atlantic/Stanley
+- Australia/Adelaide
+- Australia/Brisbane
+- Australia/Broken\_Hill
+- Australia/Currie
+- Australia/Darwin
+- Australia/Eucla
+- Australia/Hobart
+- Australia/Lindeman
+- Australia/Lord\_Howe
+- Australia/Melbourne
+- Australia/Perth
+- Australia/Sydney
+- Canada/Atlantic
+- Canada/Central
+- Canada/Eastern
+- Canada/Mountain
+- Canada/Newfoundland
+- Canada/Pacific
+- Europe/Amsterdam
+- Europe/Andorra
+- Europe/Athens
+- Europe/Belgrade
+- Europe/Berlin
+- Europe/Bratislava
+- Europe/Brussels
+- Europe/Bucharest
+- Europe/Budapest
+- Europe/Busingen
+- Europe/Chisinau
+- Europe/Copenhagen
+- Europe/Dublin
+- Europe/Gibraltar
+- Europe/Guernsey
+- Europe/Helsinki
+- Europe/Isle\_of\_Man
+- Europe/Istanbul
+- Europe/Jersey
+- Europe/Kaliningrad
+- Europe/Kiev
+- Europe/Lisbon
+- Europe/Ljubljana
+- Europe/London
+- Europe/Luxembourg
+- Europe/Madrid
+- Europe/Malta
+- Europe/Mariehamn
+- Europe/Minsk
+- Europe/Monaco
+- Europe/Moscow
+- Europe/Oslo
+- Europe/Paris
+- Europe/Podgorica
+- Europe/Prague
+- Europe/Riga
+- Europe/Rome
+- Europe/Samara
+- Europe/San\_Marino
+- Europe/Sarajevo
+- Europe/Simferopol
+- Europe/Skopje
+- Europe/Sofia
+- Europe/Stockholm
+- Europe/Tallinn
+- Europe/Tirane
+- Europe/Uzhgorod
+- Europe/Vaduz
+- Europe/Vatican
+- Europe/Vienna
+- Europe/Vilnius
+- Europe/Volgograd
+- Europe/Warsaw
+- Europe/Zagreb
+- Europe/Zaporozhye
+- Europe/Zurich
+- GMT
+- Indian/Antananarivo
+- Indian/Chagos
+- Indian/Christmas
+- Indian/Cocos
+- Indian/Comoro
+- Indian/Kerguelen
+- Indian/Mahe
+- Indian/Maldives
+- Indian/Mauritius
+- Indian/Mayotte
+- Indian/Reunion
+- Pacific/Apia
+- Pacific/Auckland
+- Pacific/Bougainville
+- Pacific/Chatham
+- Pacific/Chuuk
+- Pacific/Easter
+- Pacific/Efate
+- Pacific/Enderbury
+- Pacific/Fakaofo
+- Pacific/Fiji
+- Pacific/Funafuti
+- Pacific/Galapagos
+- Pacific/Gambier
+- Pacific/Guadalcanal
+- Pacific/Guam
+- Pacific/Honolulu
+- Pacific/Johnston
+- Pacific/Kiritimati
+- Pacific/Kosrae
+- Pacific/Kwajalein
+- Pacific/Majuro
+- Pacific/Marquesas
+- Pacific/Midway
+- Pacific/Nauru
+- Pacific/Niue
+- Pacific/Norfolk
+- Pacific/Noumea
+- Pacific/Pago\_Pago
+- Pacific/Palau
+- Pacific/Pitcairn
+- Pacific/Pohnpei
+- Pacific/Port\_Moresby
+- Pacific/Rarotonga
+- Pacific/Saipan
+- Pacific/Tahiti
+- Pacific/Tarawa
+- Pacific/Tongatapu
+- Pacific/Wake
+- Pacific/Wallis
+- US/Alaska
+- US/Arizona
+- US/Central
+- US/Eastern
+- US/Hawaii
+- US/Mountain
+- US/Pacific
+- UTC
\ No newline at end of file
--- /dev/null
+/*
+ * argparse.h - Module for parsing command line arguments.
+ */
+#ifndef ARGPARSE_H_
+#define ARGPARSE_H_
+
+#define ARGPARSE_SUCCESS 0
+#define ARGPARSE_ERR_MISSING_VALUE -1
+#define ARGPARSE_ERR_UNRECOGNIZED_ARGUMENT -2
+#define ARGPARSE_ERR_AMBIGUOUS_OPTION -3
+#define ARGPARSE_ERR_INVALID_CHOICE -4
+#define ARGPARSE_ERR_ARGUMENT_TOO_LONG -5
+#define ARGPARSE_ERR_UNHANDLED -9
+
+#define ARGPARSE_MAX_LEN_ARG 255
+
+typedef enum {
+ ARGPARSE_STORE_TRUE,
+ ARGPARSE_STORE_FALSE,
+ ARGPARSE_STORE_STRING,
+ ARGPARSE_STORE_INT,
+ ARGPARSE_STORE_STR_CHOICE
+} argparse_action_t;
+
+typedef struct argparse_args_s argparse_args_t;
+typedef struct argparse_parser_s argparse_parser_t;
+typedef struct argparse_argument_s argparse_argument_t;
+
+#include <inttypes.h>
+
+void argparse_init(argparse_parser_t * parser);
+void argparse_free(argparse_parser_t * parser);
+void argparse_add_argument(
+ argparse_parser_t * parser,
+ argparse_argument_t * argument);
+void argparse_parse(argparse_parser_t *parser, int argc, char *argv[]);
+
+struct argparse_argument_s
+{
+ char * name;
+ char shortcut;
+ char * help;
+ argparse_action_t action;
+ int32_t default_int32_t;
+ int32_t * pt_value_int32_t;
+ char * str_default;
+ char * str_value;
+ char * choices; /* choices must be separated by commas */
+};
+
+struct argparse_args_s
+{
+ argparse_argument_t * argument;
+ argparse_args_t * next;
+};
+
+struct argparse_parser_s
+{
+ argparse_args_t * args;
+ int32_t show_help;
+ argparse_argument_t help;
+};
+
+#endif /* ARGPARSE_H_ */
--- /dev/null
+/*
+ * base64.h - Base64 encoding/decoding
+ */
+#ifndef SIRI_BASE64_H_
+#define SIRI_BASE64_H_
+
+#include <stddef.h>
+
+char * base64_decode(const void * data, size_t n, size_t * size);
+char * base64_encode(const void * data, size_t n, size_t * size);
+
+
+#endif /* SIRI_BASE64_H_ */
--- /dev/null
+/*
+ * cexpr.h - Conditional expressions.
+ */
+#ifndef CEXPR_H_
+#define CEXPR_H_
+
+#define CEXPR_MAX_CURLY_DEPTH 6
+
+typedef enum
+{
+ CEXPR_EQ, /* equal */
+ CEXPR_NE, /* not equal */
+ CEXPR_GT, /* greater than */
+ CEXPR_LT, /* less than */
+ CEXPR_GE, /* greater than or equal to */
+ CEXPR_LE, /* less than or equal to */
+ CEXPR_IN, /* contains (string) */
+ CEXPR_NI, /* not contains (string) */
+ CEXPR_AND,
+ CEXPR_OR,
+} cexpr_operator_t;
+
+typedef union cexpr_via_u cexpr_via_t;
+typedef struct cexpr_condition_s cexpr_condition_t;
+typedef struct cexpr_s cexpr_t;
+typedef struct cexpr_list_s cexpr_list_t;
+
+#include <inttypes.h>
+#include <qpack/qpack.h>
+#include <cleri/cleri.h>
+
+typedef int (*cexpr_cb_t)(void * obj, cexpr_condition_t * cond);
+typedef int (*cexpr_cb_prop_t)(uint32_t prop);
+
+cexpr_t * cexpr_from_node(cleri_node_t * node);
+int cexpr_int_cmp(
+ const cexpr_operator_t operator,
+ const int64_t a,
+ const int64_t b);
+int cexpr_double_cmp(
+ const cexpr_operator_t operator,
+ const double a,
+ const double b);
+int cexpr_str_cmp(
+ const cexpr_operator_t operator,
+ const char * a,
+ const char * b);
+int cexpr_bool_cmp(
+ const cexpr_operator_t operator,
+ const int64_t a,
+ const int64_t b);
+int cexpr_run(cexpr_t * cexpr, cexpr_cb_t cb, void * obj);
+int cexpr_contains(cexpr_t * cexpr, cexpr_cb_prop_t cb);
+void cexpr_free(cexpr_t * cexpr);
+cexpr_operator_t cexpr_operator_fn(cleri_node_t * node);
+
+union cexpr_via_u
+{
+ cexpr_condition_t * cond;
+ cexpr_t * cexpr;
+};
+
+struct cexpr_condition_s
+{
+ uint32_t prop;
+ cexpr_operator_t operator;
+ int64_t int64;
+ char * str;
+};
+
+struct cexpr_list_s
+{
+ size_t len;
+ cexpr_t * cexpr[CEXPR_MAX_CURLY_DEPTH];
+};
+
+struct cexpr_s
+{
+ cexpr_operator_t operator; /* AND/OR */
+ int8_t tp_a;
+ int8_t tp_b;
+ cexpr_via_t via_a;
+ cexpr_via_t via_b;
+};
+
+#endif /* CEXPR_H_ */
--- /dev/null
+/*
+ * cfgparser.h - Reading (and later writing) to INI style files.
+ */
+#ifndef CFGPARSER_H_
+#define CFGPARSER_H_
+
+typedef enum
+{
+ CFGPARSER_SUCCESS,
+ CFGPARSER_ERR_READING_FILE,
+ CFGPARSER_ERR_SESSION_NOT_OPEN,
+ CFGPARSER_ERR_MISSING_EQUAL_SIGN,
+ CFGPARSER_ERR_OPTION_ALREADY_DEFINED,
+ CFGPARSER_ERR_SECTION_NOT_FOUND,
+ CFGPARSER_ERR_OPTION_NOT_FOUND
+} cfgparser_return_t;
+
+typedef enum
+{
+ CFGPARSER_TP_INTEGER,
+ CFGPARSER_TP_STRING,
+ CFGPARSER_TP_REAL,
+} cfgparser_tp_t;
+
+typedef union cfgparser_u cfgparser_via_t;
+typedef struct cfgparser_option_s cfgparser_option_t;
+typedef struct cfgparser_section_s cfgparser_section_t;
+typedef struct cfgparser_s cfgparser_t;
+
+#include <inttypes.h>
+
+cfgparser_return_t cfgparser_read(cfgparser_t * cfgparser, const char * fn);
+cfgparser_t * cfgparser_new(void);
+void cfgparser_free(cfgparser_t * cfgparser);
+cfgparser_section_t * cfgparser_section(
+ cfgparser_t * cfgparser,
+ const char * name);
+cfgparser_option_t * cfgparser_string_option(
+ cfgparser_section_t * section,
+ const char * name,
+ const char * val,
+ const char * def);
+cfgparser_option_t * cfgparser_integer_option(
+ cfgparser_section_t * section,
+ const char * name,
+ int32_t val,
+ int32_t def);
+cfgparser_option_t * cfgparser_real_option(
+ cfgparser_section_t * section,
+ const char * name,
+ double val,
+ double def);
+const char * cfgparser_errmsg(cfgparser_return_t err);
+cfgparser_return_t cfgparser_get_section(
+ cfgparser_section_t ** section,
+ cfgparser_t * cfgparser,
+ const char * section_name);
+cfgparser_return_t cfgparser_get_option(
+ cfgparser_option_t ** option,
+ cfgparser_t * cfgparser,
+ const char * section_name,
+ const char * option_name);
+
+
+union cfgparser_u
+{
+ int32_t integer;
+ double real;
+ char * string;
+};
+
+struct cfgparser_option_s
+{
+ char * name;
+ cfgparser_tp_t tp;
+ cfgparser_via_t * val;
+ cfgparser_via_t * def;
+ struct cfgparser_option_s * next;
+};
+
+struct cfgparser_section_s
+{
+ char * name;
+ cfgparser_option_t * options;
+ struct cfgparser_section_s * next;
+};
+
+struct cfgparser_s
+{
+ cfgparser_section_t * sections;
+};
+
+#endif /* CFGPARSER_H_ */
--- /dev/null
+/*
+ * ctree.h - Compact Binary Tree implementation.
+ */
+#ifndef CTREE_H_
+#define CTREE_H_
+
+enum
+{
+ CT_ERR=-1,
+ CT_OK,
+ CT_EXISTS,
+};
+
+typedef struct ct_s ct_t;
+typedef struct ct_node_s ct_node_t;
+typedef struct ct_node_s * ct_nodes_t[32];
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdio.h>
+
+typedef int (*ct_item_cb)(
+ const char * key,
+ size_t len,
+ void * data,
+ void * args);
+typedef int (*ct_val_cb)(void * data, void * args);
+typedef void (*ct_free_cb)(void * data);
+
+ct_t * ct_new(void);
+void ct_free(ct_t * ct, ct_free_cb cb);
+int ct_add(ct_t * ct, const char * key, void * data);
+void * ct_get(ct_t * node, const char * key);
+void ** ct_getaddr(ct_t * ct, const char * key);
+void * ct_getn(ct_t * ct, const char * key, size_t n);
+void * ct_pop(ct_t * ct, const char * key);
+int ct_items(ct_t * ct, ct_item_cb cb, void * args);
+int ct_values(ct_t * ct, ct_val_cb cb, void * args);
+void ct_valuesn(ct_t * ct, size_t * n, ct_val_cb cb, void * args);
+
+struct ct_node_s
+{
+ uint8_t offset;
+ uint8_t n;
+ uint8_t size;
+ uint8_t pad0;
+ uint32_t len;
+ ct_nodes_t * nodes;
+ char * key;
+ void * data;
+};
+
+struct ct_s
+{
+ uint8_t offset;
+ uint8_t n;
+ uint16_t pad0;
+ uint32_t len;
+ ct_nodes_t * nodes;
+};
+
+#endif /* CTREE_H_ */
--- /dev/null
+/*
+ * expr.h - For parsing expressions.
+ */
+#ifndef EXPR_H_
+#define EXPR_H_
+
+#define EXPR_DIVISION_BY_ZERO -1
+#define EXPR_MODULO_BY_ZERO -2
+
+/* values below are used by SiriDB and are not necessary for this library */
+#define EXPR_TOO_LONG -3
+#define EXPR_TIME_OUT_OF_RANGE -4
+#define EXPR_INVALID_DATE_STRING -5
+#define EXPR_MEM_ALLOC_ERR -6
+
+/*
+ * Max size for expressions
+ * (not the total query, just an expression in the query
+ */
+#define EXPR_MAX_SIZE 512
+
+#include <inttypes.h>
+
+/* Returns 0 when result is successful set */
+int expr_parse(int64_t * result, const char * expr);
+
+
+#endif /* EXPR_H_ */
--- /dev/null
+/*
+ * imap.h - Lookup map for uint64_t integer keys with set operation support.
+ */
+#ifndef IMAP_H_
+#define IMAP_H_
+
+typedef struct imap_node_s imap_node_t;
+typedef struct imap_s imap_t;
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <vec/vec.h>
+
+typedef int (*imap_cb)(void * data, void * args);
+typedef void (*imap_free_cb)(void * data);
+
+typedef void (*imap_update_cb)(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb);
+
+imap_t * imap_new(void);
+void imap_free(imap_t * imap, imap_free_cb cb);
+int imap_set(imap_t * imap, uint64_t id, void * data);
+int imap_add(imap_t * imap, uint64_t id, void * data);
+void * imap_get(imap_t * imap, uint64_t id);
+void * imap_pop(imap_t * imap, uint64_t id);
+int imap_walk(imap_t * imap, imap_cb cb, void * data);
+void imap_walkn(imap_t * imap, size_t * n, imap_cb cb, void * data);
+vec_t * imap_vec(imap_t * imap);
+vec_t * imap_vec_pop(imap_t * imap);
+vec_t * imap_2vec(imap_t * imap);
+vec_t * imap_2vec_ref(imap_t * imap);
+void imap_union_ref(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb);
+void imap_intersection_ref(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb);
+void imap_difference_ref(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb);
+void imap_symmetric_difference_ref(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb);
+
+
+struct imap_node_s
+{
+ size_t size;
+ void * data ;
+ imap_node_t * nodes;
+};
+
+struct imap_s
+{
+ size_t len;
+ vec_t * vec;
+ imap_node_t nodes[];
+};
+
+#endif /* IMAP_H_ */
--- /dev/null
+/*
+ * iso8601.h - Library to parse ISO 8601 dates. Time-zones are found with
+ * tzset() in either /usr/lib/zoneinfo/ or /usr/share/zoneinfo/.
+ */
+#ifndef ISO8601_H_
+#define ISO8601_H_
+
+#include <inttypes.h>
+
+typedef int16_t iso8601_tz_t;
+
+/* Returns a time-zone identifier from a given name or a negative value
+ * in case of an error or when the time-zone is not found.
+ * Examples of tzname are: Europe/Amsterdam, UTC etc.
+ *
+ * (this function is not case-sensitive)
+ */
+iso8601_tz_t iso8601_tz(const char * tzname);
+
+/* Returns the name for a given timezone */
+const char * iso8601_tzname(iso8601_tz_t tz);
+
+/* Returns a time-stamp in seconds for a given date or a negative value in
+ * case or an error. Time-zone information can be parsed but is also allowed
+ * in the string.
+ */
+int64_t iso8601_parse_date(const char * str, iso8601_tz_t tz);
+
+#endif /* ISO8601_H_ */
--- /dev/null
+/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifndef http_parser_h
+#define http_parser_h
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Also update SONAME in the Makefile whenever you change these. */
+#define HTTP_PARSER_VERSION_MAJOR 2
+#define HTTP_PARSER_VERSION_MINOR 9
+#define HTTP_PARSER_VERSION_PATCH 2
+
+#include <stddef.h>
+#if defined(_WIN32) && !defined(__MINGW32__) && \
+ (!defined(_MSC_VER) || _MSC_VER<1600) && !defined(__WINE__)
+#include <BaseTsd.h>
+typedef __int8 int8_t;
+typedef unsigned __int8 uint8_t;
+typedef __int16 int16_t;
+typedef unsigned __int16 uint16_t;
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include <stdint.h>
+#endif
+
+/* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run
+ * faster
+ */
+#ifndef HTTP_PARSER_STRICT
+# define HTTP_PARSER_STRICT 1
+#endif
+
+/* Maximium header size allowed. If the macro is not defined
+ * before including this header then the default is used. To
+ * change the maximum header size, define the macro in the build
+ * environment (e.g. -DHTTP_MAX_HEADER_SIZE=<value>). To remove
+ * the effective limit on the size of the header, define the macro
+ * to a very large number (e.g. -DHTTP_MAX_HEADER_SIZE=0x7fffffff)
+ */
+#ifndef HTTP_MAX_HEADER_SIZE
+# define HTTP_MAX_HEADER_SIZE (80*1024)
+#endif
+
+typedef struct http_parser http_parser;
+typedef struct http_parser_settings http_parser_settings;
+
+
+/* Callbacks should return non-zero to indicate an error. The parser will
+ * then halt execution.
+ *
+ * The one exception is on_headers_complete. In a HTTP_RESPONSE parser
+ * returning '1' from on_headers_complete will tell the parser that it
+ * should not expect a body. This is used when receiving a response to a
+ * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding:
+ * chunked' headers that indicate the presence of a body.
+ *
+ * Returning `2` from on_headers_complete will tell parser that it should not
+ * expect neither a body nor any futher responses on this connection. This is
+ * useful for handling responses to a CONNECT request which may not contain
+ * `Upgrade` or `Connection: upgrade` headers.
+ *
+ * http_data_cb does not return data chunks. It will be called arbitrarily
+ * many times for each string. E.G. you might get 10 callbacks for "on_url"
+ * each providing just a few characters more data.
+ */
+typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);
+typedef int (*http_cb) (http_parser*);
+
+
+/* Status Codes */
+#define HTTP_STATUS_MAP(XX) \
+ XX(100, CONTINUE, Continue) \
+ XX(101, SWITCHING_PROTOCOLS, Switching Protocols) \
+ XX(102, PROCESSING, Processing) \
+ XX(200, OK, OK) \
+ XX(201, CREATED, Created) \
+ XX(202, ACCEPTED, Accepted) \
+ XX(203, NON_AUTHORITATIVE_INFORMATION, Non-Authoritative Information) \
+ XX(204, NO_CONTENT, No Content) \
+ XX(205, RESET_CONTENT, Reset Content) \
+ XX(206, PARTIAL_CONTENT, Partial Content) \
+ XX(207, MULTI_STATUS, Multi-Status) \
+ XX(208, ALREADY_REPORTED, Already Reported) \
+ XX(226, IM_USED, IM Used) \
+ XX(300, MULTIPLE_CHOICES, Multiple Choices) \
+ XX(301, MOVED_PERMANENTLY, Moved Permanently) \
+ XX(302, FOUND, Found) \
+ XX(303, SEE_OTHER, See Other) \
+ XX(304, NOT_MODIFIED, Not Modified) \
+ XX(305, USE_PROXY, Use Proxy) \
+ XX(307, TEMPORARY_REDIRECT, Temporary Redirect) \
+ XX(308, PERMANENT_REDIRECT, Permanent Redirect) \
+ XX(400, BAD_REQUEST, Bad Request) \
+ XX(401, UNAUTHORIZED, Unauthorized) \
+ XX(402, PAYMENT_REQUIRED, Payment Required) \
+ XX(403, FORBIDDEN, Forbidden) \
+ XX(404, NOT_FOUND, Not Found) \
+ XX(405, METHOD_NOT_ALLOWED, Method Not Allowed) \
+ XX(406, NOT_ACCEPTABLE, Not Acceptable) \
+ XX(407, PROXY_AUTHENTICATION_REQUIRED, Proxy Authentication Required) \
+ XX(408, REQUEST_TIMEOUT, Request Timeout) \
+ XX(409, CONFLICT, Conflict) \
+ XX(410, GONE, Gone) \
+ XX(411, LENGTH_REQUIRED, Length Required) \
+ XX(412, PRECONDITION_FAILED, Precondition Failed) \
+ XX(413, PAYLOAD_TOO_LARGE, Payload Too Large) \
+ XX(414, URI_TOO_LONG, URI Too Long) \
+ XX(415, UNSUPPORTED_MEDIA_TYPE, Unsupported Media Type) \
+ XX(416, RANGE_NOT_SATISFIABLE, Range Not Satisfiable) \
+ XX(417, EXPECTATION_FAILED, Expectation Failed) \
+ XX(421, MISDIRECTED_REQUEST, Misdirected Request) \
+ XX(422, UNPROCESSABLE_ENTITY, Unprocessable Entity) \
+ XX(423, LOCKED, Locked) \
+ XX(424, FAILED_DEPENDENCY, Failed Dependency) \
+ XX(426, UPGRADE_REQUIRED, Upgrade Required) \
+ XX(428, PRECONDITION_REQUIRED, Precondition Required) \
+ XX(429, TOO_MANY_REQUESTS, Too Many Requests) \
+ XX(431, REQUEST_HEADER_FIELDS_TOO_LARGE, Request Header Fields Too Large) \
+ XX(451, UNAVAILABLE_FOR_LEGAL_REASONS, Unavailable For Legal Reasons) \
+ XX(500, INTERNAL_SERVER_ERROR, Internal Server Error) \
+ XX(501, NOT_IMPLEMENTED, Not Implemented) \
+ XX(502, BAD_GATEWAY, Bad Gateway) \
+ XX(503, SERVICE_UNAVAILABLE, Service Unavailable) \
+ XX(504, GATEWAY_TIMEOUT, Gateway Timeout) \
+ XX(505, HTTP_VERSION_NOT_SUPPORTED, HTTP Version Not Supported) \
+ XX(506, VARIANT_ALSO_NEGOTIATES, Variant Also Negotiates) \
+ XX(507, INSUFFICIENT_STORAGE, Insufficient Storage) \
+ XX(508, LOOP_DETECTED, Loop Detected) \
+ XX(510, NOT_EXTENDED, Not Extended) \
+ XX(511, NETWORK_AUTHENTICATION_REQUIRED, Network Authentication Required) \
+
+enum http_status
+ {
+#define XX(num, name, string) HTTP_STATUS_##name = num,
+ HTTP_STATUS_MAP(XX)
+#undef XX
+ };
+
+
+/* Request Methods */
+#define HTTP_METHOD_MAP(XX) \
+ XX(0, DELETE, DELETE) \
+ XX(1, GET, GET) \
+ XX(2, HEAD, HEAD) \
+ XX(3, POST, POST) \
+ XX(4, PUT, PUT) \
+ /* pathological */ \
+ XX(5, CONNECT, CONNECT) \
+ XX(6, OPTIONS, OPTIONS) \
+ XX(7, TRACE, TRACE) \
+ /* WebDAV */ \
+ XX(8, COPY, COPY) \
+ XX(9, LOCK, LOCK) \
+ XX(10, MKCOL, MKCOL) \
+ XX(11, MOVE, MOVE) \
+ XX(12, PROPFIND, PROPFIND) \
+ XX(13, PROPPATCH, PROPPATCH) \
+ XX(14, SEARCH, SEARCH) \
+ XX(15, UNLOCK, UNLOCK) \
+ XX(16, BIND, BIND) \
+ XX(17, REBIND, REBIND) \
+ XX(18, UNBIND, UNBIND) \
+ XX(19, ACL, ACL) \
+ /* subversion */ \
+ XX(20, REPORT, REPORT) \
+ XX(21, MKACTIVITY, MKACTIVITY) \
+ XX(22, CHECKOUT, CHECKOUT) \
+ XX(23, MERGE, MERGE) \
+ /* upnp */ \
+ XX(24, MSEARCH, M-SEARCH) \
+ XX(25, NOTIFY, NOTIFY) \
+ XX(26, SUBSCRIBE, SUBSCRIBE) \
+ XX(27, UNSUBSCRIBE, UNSUBSCRIBE) \
+ /* RFC-5789 */ \
+ XX(28, PATCH, PATCH) \
+ XX(29, PURGE, PURGE) \
+ /* CalDAV */ \
+ XX(30, MKCALENDAR, MKCALENDAR) \
+ /* RFC-2068, section 19.6.1.2 */ \
+ XX(31, LINK, LINK) \
+ XX(32, UNLINK, UNLINK) \
+ /* icecast */ \
+ XX(33, SOURCE, SOURCE) \
+
+enum http_method
+ {
+#define XX(num, name, string) HTTP_##name = num,
+ HTTP_METHOD_MAP(XX)
+#undef XX
+ };
+
+
+enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
+
+
+/* Flag values for http_parser.flags field */
+enum flags
+ { F_CHUNKED = 1 << 0
+ , F_CONNECTION_KEEP_ALIVE = 1 << 1
+ , F_CONNECTION_CLOSE = 1 << 2
+ , F_CONNECTION_UPGRADE = 1 << 3
+ , F_TRAILING = 1 << 4
+ , F_UPGRADE = 1 << 5
+ , F_SKIPBODY = 1 << 6
+ , F_CONTENTLENGTH = 1 << 7
+ };
+
+
+/* Map for errno-related constants
+ *
+ * The provided argument should be a macro that takes 2 arguments.
+ */
+#define HTTP_ERRNO_MAP(XX) \
+ /* No error */ \
+ XX(OK, "success") \
+ \
+ /* Callback-related errors */ \
+ XX(CB_message_begin, "the on_message_begin callback failed") \
+ XX(CB_url, "the on_url callback failed") \
+ XX(CB_header_field, "the on_header_field callback failed") \
+ XX(CB_header_value, "the on_header_value callback failed") \
+ XX(CB_headers_complete, "the on_headers_complete callback failed") \
+ XX(CB_body, "the on_body callback failed") \
+ XX(CB_message_complete, "the on_message_complete callback failed") \
+ XX(CB_status, "the on_status callback failed") \
+ XX(CB_chunk_header, "the on_chunk_header callback failed") \
+ XX(CB_chunk_complete, "the on_chunk_complete callback failed") \
+ \
+ /* Parsing-related errors */ \
+ XX(INVALID_EOF_STATE, "stream ended at an unexpected time") \
+ XX(HEADER_OVERFLOW, \
+ "too many header bytes seen; overflow detected") \
+ XX(CLOSED_CONNECTION, \
+ "data received after completed connection: close message") \
+ XX(INVALID_VERSION, "invalid HTTP version") \
+ XX(INVALID_STATUS, "invalid HTTP status code") \
+ XX(INVALID_METHOD, "invalid HTTP method") \
+ XX(INVALID_URL, "invalid URL") \
+ XX(INVALID_HOST, "invalid host") \
+ XX(INVALID_PORT, "invalid port") \
+ XX(INVALID_PATH, "invalid path") \
+ XX(INVALID_QUERY_STRING, "invalid query string") \
+ XX(INVALID_FRAGMENT, "invalid fragment") \
+ XX(LF_EXPECTED, "LF character expected") \
+ XX(INVALID_HEADER_TOKEN, "invalid character in header") \
+ XX(INVALID_CONTENT_LENGTH, \
+ "invalid character in content-length header") \
+ XX(UNEXPECTED_CONTENT_LENGTH, \
+ "unexpected content-length header") \
+ XX(INVALID_CHUNK_SIZE, \
+ "invalid character in chunk size header") \
+ XX(INVALID_CONSTANT, "invalid constant string") \
+ XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\
+ XX(STRICT, "strict mode assertion failed") \
+ XX(PAUSED, "parser is paused") \
+ XX(UNKNOWN, "an unknown error occurred")
+
+
+/* Define HPE_* values for each errno value above */
+#define HTTP_ERRNO_GEN(n, s) HPE_##n,
+enum http_errno {
+ HTTP_ERRNO_MAP(HTTP_ERRNO_GEN)
+};
+#undef HTTP_ERRNO_GEN
+
+
+/* Get an http_errno value from an http_parser */
+#define HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno)
+
+
+struct http_parser {
+ /** PRIVATE **/
+ unsigned int type : 2; /* enum http_parser_type */
+ unsigned int flags : 8; /* F_* values from 'flags' enum; semi-public */
+ unsigned int state : 7; /* enum state from http_parser.c */
+ unsigned int header_state : 7; /* enum header_state from http_parser.c */
+ unsigned int index : 7; /* index into current matcher */
+ unsigned int lenient_http_headers : 1;
+
+ uint32_t nread; /* # bytes read in various scenarios */
+ uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */
+
+ /** READ-ONLY **/
+ unsigned short http_major;
+ unsigned short http_minor;
+ unsigned int status_code : 16; /* responses only */
+ unsigned int method : 8; /* requests only */
+ unsigned int http_errno : 7;
+
+ /* 1 = Upgrade header was present and the parser has exited because of that.
+ * 0 = No upgrade header present.
+ * Should be checked when http_parser_execute() returns in addition to
+ * error checking.
+ */
+ unsigned int upgrade : 1;
+
+ /** PUBLIC **/
+ void *data; /* A pointer to get hook to the "connection" or "socket" object */
+};
+
+
+struct http_parser_settings {
+ http_cb on_message_begin;
+ http_data_cb on_url;
+ http_data_cb on_status;
+ http_data_cb on_header_field;
+ http_data_cb on_header_value;
+ http_cb on_headers_complete;
+ http_data_cb on_body;
+ http_cb on_message_complete;
+ /* When on_chunk_header is called, the current chunk length is stored
+ * in parser->content_length.
+ */
+ http_cb on_chunk_header;
+ http_cb on_chunk_complete;
+};
+
+
+enum http_parser_url_fields
+ { UF_SCHEMA = 0
+ , UF_HOST = 1
+ , UF_PORT = 2
+ , UF_PATH = 3
+ , UF_QUERY = 4
+ , UF_FRAGMENT = 5
+ , UF_USERINFO = 6
+ , UF_MAX = 7
+ };
+
+
+/* Result structure for http_parser_parse_url().
+ *
+ * Callers should index into field_data[] with UF_* values iff field_set
+ * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and
+ * because we probably have padding left over), we convert any port to
+ * a uint16_t.
+ */
+struct http_parser_url {
+ uint16_t field_set; /* Bitmask of (1 << UF_*) values */
+ uint16_t port; /* Converted UF_PORT string */
+
+ struct {
+ uint16_t off; /* Offset into buffer in which field starts */
+ uint16_t len; /* Length of run in buffer */
+ } field_data[UF_MAX];
+};
+
+
+/* Returns the library version. Bits 16-23 contain the major version number,
+ * bits 8-15 the minor version number and bits 0-7 the patch level.
+ * Usage example:
+ *
+ * unsigned long version = http_parser_version();
+ * unsigned major = (version >> 16) & 255;
+ * unsigned minor = (version >> 8) & 255;
+ * unsigned patch = version & 255;
+ * printf("http_parser v%u.%u.%u\n", major, minor, patch);
+ */
+unsigned long http_parser_version(void);
+
+void http_parser_init(http_parser *parser, enum http_parser_type type);
+
+
+/* Initialize http_parser_settings members to 0
+ */
+void http_parser_settings_init(http_parser_settings *settings);
+
+
+/* Executes the parser. Returns number of parsed bytes. Sets
+ * `parser->http_errno` on error. */
+size_t http_parser_execute(http_parser *parser,
+ const http_parser_settings *settings,
+ const char *data,
+ size_t len);
+
+
+/* If http_should_keep_alive() in the on_headers_complete or
+ * on_message_complete callback returns 0, then this should be
+ * the last message on the connection.
+ * If you are the server, respond with the "Connection: close" header.
+ * If you are the client, close the connection.
+ */
+int http_should_keep_alive(const http_parser *parser);
+
+/* Returns a string version of the HTTP method. */
+const char *http_method_str(enum http_method m);
+
+/* Returns a string version of the HTTP status code. */
+const char *http_status_str(enum http_status s);
+
+/* Return a string name of the given error */
+const char *http_errno_name(enum http_errno err);
+
+/* Return a string description of the given error */
+const char *http_errno_description(enum http_errno err);
+
+/* Initialize all http_parser_url members to 0 */
+void http_parser_url_init(struct http_parser_url *u);
+
+/* Parse a URL; return nonzero on failure */
+int http_parser_parse_url(const char *buf, size_t buflen,
+ int is_connect,
+ struct http_parser_url *u);
+
+/* Pause or un-pause the parser; a nonzero value pauses */
+void http_parser_pause(http_parser *parser, int paused);
+
+/* Checks if this is the final chunk of the body. */
+int http_body_is_final(const http_parser *parser);
+
+/* Change the maximum header size provided at compile time. */
+void http_parser_set_max_header_size(uint32_t size);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
--- /dev/null
+/*
+ * llist.h - Linked List implementation.
+ */
+#ifndef LLIST_H_
+#define LLIST_H_
+
+typedef struct llist_s llist_t;
+typedef struct llist_node_s llist_node_t;
+
+#include <stdio.h>
+#include <vec/vec.h>
+
+typedef int (*llist_cb)(void * data, void * args);
+typedef void (*llist_destroy_cb)(void * data);
+
+llist_t * llist_new(void);
+void llist_free_cb(llist_t * llist, llist_cb cb, void * args);
+void llist_destroy(llist_t * llist, llist_destroy_cb cb);
+int llist_append(llist_t * llist, void * data);
+int llist_walk(llist_t * llist, llist_cb cb, void * args);
+void llist_walkn(llist_t * llist, size_t * n, llist_cb cb, void * args);
+vec_t * llist2vec(llist_t * llist);
+void * llist_get(llist_t * llist, llist_cb cb, void * args);
+void * llist_remove(llist_t * llist, llist_cb cb, void * args);
+void * llist_pop(llist_t * llist);
+void * llist_shift(llist_t * llist);
+
+struct llist_node_s
+{
+ void * data;
+ llist_node_t * next;
+};
+
+struct llist_s
+{
+ size_t len;
+ llist_node_t * first;
+ llist_node_t * last;
+};
+
+#endif /* LLIST_H_ */
--- /dev/null
+/*
+ * lock.h - Lock a directory by using a lock file.
+ */
+#ifndef LOCK_H_
+#define LOCK_H_
+
+#define LOCK_QUIT_IF_EXIST 1
+
+typedef enum
+{
+ LOCK_IS_LOCKED_ERR=-6,
+ LOCK_PROCESS_NAME_ERR,
+ LOCK_WRITE_ERR,
+ LOCK_READ_ERR,
+ LOCK_UNLINK_ERR,
+ LOCK_MEM_ALLOC_ERR,
+ LOCK_REMOVED=0,
+ LOCK_NEW,
+ LOCK_OVERWRITE
+} lock_t;
+
+lock_t lock_lock(const char * path, int flags);
+lock_t lock_unlock(const char * path);
+const char * lock_str(lock_t rc);
+
+#endif /* LOCK_H_ */
--- /dev/null
+/*
+ * logger.h - Logging module.
+ */
+#ifndef LOGGER_H_
+#define LOGGER_H_
+
+#include <stdio.h>
+#ifdef __APPLE__
+typedef struct __sFILE LOGGER_IO_FILE;
+#else
+typedef struct _IO_FILE LOGGER_IO_FILE;
+#endif
+
+#define LOGGER_DEBUG 0
+#define LOGGER_INFO 1
+#define LOGGER_WARNING 2
+#define LOGGER_ERROR 3
+#define LOGGER_CRITICAL 4
+
+#define LOGGER_NUM_LEVELS 5
+
+#define LOGGER_FLAG_COLORED 1
+
+typedef struct logger_s logger_t;
+
+void logger_init(LOGGER_IO_FILE * ostream, int log_level);
+void logger_set_level(int log_level);
+const char * logger_level_name(int log_level);
+
+void log__debug(const char * fmt, ...);
+void log__info(const char * fmt, ...);
+void log__warning(const char * fmt, ...);
+void log__error(const char * fmt, ...);
+void log__critical(const char * fmt, ...);
+
+extern logger_t Logger;
+
+#define log_debug(fmt, ...) \
+ if (Logger.level == LOGGER_DEBUG) \
+ log__debug(fmt, ##__VA_ARGS__) \
+
+#define log_info(fmt, ...) \
+ if (Logger.level <= LOGGER_INFO) \
+ log__info(fmt, ##__VA_ARGS__) \
+
+#define log_warning(fmt, ...) \
+ if (Logger.level <= LOGGER_WARNING) \
+ log__warning(fmt, ##__VA_ARGS__) \
+
+#define log_error(fmt, ...) \
+ if (Logger.level <= LOGGER_ERROR) \
+ log__error(fmt, ##__VA_ARGS__) \
+
+#define log_critical(fmt, ...) \
+ if (Logger.level <= LOGGER_CRITICAL) \
+ log__critical(fmt, ##__VA_ARGS__) \
+
+#define LOGC(fmt, ...) \
+ fprintf(Logger.ostream, "%s:%d ", __FILE__, __LINE__); \
+ log_critical(fmt, ##__VA_ARGS__)
+
+struct logger_s
+{
+ LOGGER_IO_FILE * ostream;
+ int level;
+ const char * level_name;
+ int flags;
+};
+
+#endif /* LOGGER_H_ */
--- /dev/null
+/*
+ * util/omap.h
+ */
+#ifndef OMAP_H_
+#define OMAP_H_
+
+enum
+{
+ OMAP_ERR_EXIST =-2,
+ OMAP_ERR_ALLOC =-1,
+ OMAP_SUCCESS =0
+};
+
+typedef struct omap_s omap_t;
+typedef struct omap__s omap__t;
+typedef struct omap__s * omap_iter_t;
+
+#include <inttypes.h>
+
+typedef void (*omap_destroy_cb)(void * data);
+
+/* private */
+struct omap__s
+{
+ omap__t * next_;
+ uint64_t id_;
+ void * data_;
+};
+
+omap_t * omap_create(void);
+void omap_destroy(omap_t * omap, omap_destroy_cb cb);
+int omap_add(omap_t * omap, uint64_t id, void * data);
+void * omap_set(omap_t * omap, uint64_t id, void * data);
+void * omap_get(omap_t * omap, uint64_t id);
+uint64_t * omap_last_id(omap_t * omap);
+void * omap_rm(omap_t * omap, uint64_t id);
+static inline omap_iter_t omap_iter(omap_t * omap);
+static inline uint64_t omap_iter_id(omap_iter_t iter);
+#define omap_loop(iter__, var__) \
+ ;iter__ && \
+ (var__ = iter__->data_); \
+ iter__ = iter__->next_
+
+struct omap_s
+{
+ omap__t * next_;
+ size_t n;
+};
+
+static inline omap_iter_t omap_iter(omap_t * omap)
+{
+ return omap->next_;
+}
+
+static inline uint64_t omap_iter_id(omap_iter_t iter)
+{
+ return iter->id_;
+}
+
+
+#endif /* OMAP_H_ */
--- /dev/null
+/*
+ * owcrypt.h - One Way Encryption. (used for storing a database user password)
+ */
+#ifndef OWCRYPT_H_
+#define OWCRYPT_H_
+
+#define OWCRYPT_SZ 64
+#define OWCRYPT_SALT_SZ 10
+
+void owcrypt(const char * password, const char * salt, char * encrypted);
+void owcrypt_gen_salt(char * salt);
+
+#endif /* OWCRYPT_H_ */
--- /dev/null
+/*
+ * procinfo.h - Process info for the current running process.
+ *
+ * Got most information from:
+ *
+ * http://stackoverflow.com/questions/63166/how-to-determine-cpu-and-
+ * memory-consumption-from-inside-a-process
+ *
+ */
+
+#ifndef PROCINFO_H_
+#define PROCINFO_H_
+
+/* Total_Physical_Memory returned in KB */
+long int procinfo_total_physical_memory(void);
+
+/* Total_Virtual_Memory returned in KB */
+long int procinfo_total_virtual_memory(void);
+
+/* Total Open Files */
+long int procinfo_open_files(const char * path, int include_fd);
+
+
+#endif /* PROCINFO_H_ */
--- /dev/null
+/*
+ * qpack.h - Efficient binary serialization format.
+ */
+#ifndef QPACK_H_
+#define QPACK_H_
+
+#include <inttypes.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#define QP_SUGGESTED_SIZE 65536
+
+typedef enum
+{
+ /*
+ * Values with -##- will never be returned while unpacking. For example
+ * a QP_INT8 (1 byte signed integer) will be returned as QP_INT64.
+ */
+ QP_END, /* at the end while unpacking */
+ QP_ERR, /* error */
+ QP_RAW, /* raw string */
+ /*
+ * Both END and RAW are never actually packed but 0 and 1 are reserved
+ * for positive signed integers.
+ *
+ * Fixed positive integers from 0 till 63 [ 0...63 ]
+ *
+ * Fixed negative integers from -60 till -1 [ 64...123 ]
+ *
+ */
+ QP_HOOK=124, /* Hook is not used by SiriDB */
+ QP_DOUBLE_N1=125, /* ## double value -1.0 */
+ QP_DOUBLE_0, /* ## double value 0.0 */
+ QP_DOUBLE_1, /* ## double value 1.0 */
+ /*
+ * Fixed raw strings lengths from 0 till 99 [ 128...227 ]
+ */
+ QP_RAW8=228, /* ## raw string with length < 1 byte */
+ QP_RAW16, /* ## raw string with length < 1 byte */
+ QP_RAW32, /* ## raw string with length < 1 byte */
+ QP_RAW64, /* ## raw string with length < 1 byte */
+ QP_INT8, /* ## 1 byte signed integer */
+ QP_INT16, /* ## 2 byte signed integer */
+ QP_INT32, /* ## 4 byte signed integer */
+ QP_INT64, /* 8 bytes signed integer */
+ QP_DOUBLE, /* 8 bytes double */
+ QP_ARRAY0, /* empty array */
+ QP_ARRAY1, /* array with 1 item */
+ QP_ARRAY2, /* array with 2 items */
+ QP_ARRAY3, /* array with 3 items */
+ QP_ARRAY4, /* array with 4 items */
+ QP_ARRAY5, /* array with 5 items */
+ QP_MAP0, /* empty map */
+ QP_MAP1, /* map with 1 item */
+ QP_MAP2, /* map with 2 items */
+ QP_MAP3, /* map with 3 items */
+ QP_MAP4, /* map with 4 items */
+ QP_MAP5, /* map with 5 items */
+ QP_TRUE, /* boolean true */
+ QP_FALSE, /* boolean false */
+ QP_NULL, /* null (none, nil) */
+ QP_ARRAY_OPEN, /* open a new array */
+ QP_MAP_OPEN, /* open a new map */
+ QP_ARRAY_CLOSE, /* close array */
+ QP_MAP_CLOSE, /* close map */
+} qp_types_t;
+
+typedef union qp_via_u qp_via_t;
+typedef struct qp_obj_s qp_obj_t;
+typedef struct qp_unpacker_s qp_unpacker_t;
+typedef struct qp_packer_s qp_packer_t;
+typedef FILE qp_fpacker_t;
+
+union qp_via_u
+{
+ int64_t int64;
+ uint64_t uint64;
+ double real;
+ unsigned char * raw;
+ char * str;
+};
+
+struct qp_obj_s
+{
+ uint8_t tp;
+ size_t len;
+ qp_via_t via;
+};
+
+struct qp_unpacker_s
+{
+ unsigned char * source; /* can be NULL or a copy or the source */
+ unsigned char * pt;
+ unsigned char * end;
+};
+
+struct qp_packer_s
+{
+ size_t len;
+ size_t buffer_size;
+ size_t alloc_size;
+ unsigned char * buffer;
+};
+
+
+#define qp_open fopen /* returns NULL in case of an error */
+#define qp_close fclose /* 0 if successful, EOF in case of an error */
+#define qp_flush fflush /* 0 if successful, EOF in case of an error */
+
+/* packer: create, destroy and extend functions */
+qp_packer_t * qp_packer_new(size_t alloc_size);
+void qp_packer_free(qp_packer_t * packer);
+int qp_packer_extend(qp_packer_t * packer, qp_packer_t * source);
+int qp_packer_extend_fu(qp_packer_t * packer, qp_unpacker_t * unpacker);
+
+/* unpacker: create and destroy functions */
+void qp_unpacker_init(qp_unpacker_t * unpacker, unsigned char * pt, size_t len);
+void qp_unpacker_ff_free(qp_unpacker_t * unpacker);
+qp_unpacker_t * qp_unpacker_ff(const char * fn);
+
+/* step functions to be used with an unpacker */
+qp_types_t qp_next(qp_unpacker_t * unpacker, qp_obj_t * qp_obj);
+qp_types_t qp_current(qp_unpacker_t * unpacker);
+qp_types_t qp_skip_next(qp_unpacker_t * unpacker);
+
+/* print function */
+void qp_print(unsigned char * pt, size_t len);
+
+/* Shortcut to print a packer object */
+#define qp_packer_print(packer) \
+ qp_print(packer->buffer, packer->len)
+
+/* Shortcut to print an unpacker object */
+#define qp_unpacker_print(unpacker) \
+ qp_print((unpacker)->pt, (unpacker)->end - (unpacker)->pt)
+
+/* Test functions */
+static inline int qp_is_array(qp_types_t tp)
+{
+ return tp == QP_ARRAY_OPEN || (tp >= QP_ARRAY0 && tp <= QP_ARRAY5);
+}
+static inline int qp_is_raw(qp_types_t tp)
+{
+ return tp == QP_RAW;
+}
+static inline int qp_is_map(qp_types_t tp)
+{
+ return tp == QP_MAP_OPEN || (tp >= QP_MAP0 && tp <= QP_MAP5);
+}
+static inline int qp_is_close(qp_types_t tp)
+{
+ return tp >= QP_ARRAY_CLOSE;
+}
+static inline int qp_is_int(qp_types_t tp)
+{
+ return tp == QP_INT64;
+}
+static inline int qp_is_double(qp_types_t tp)
+{
+ return tp == QP_DOUBLE;
+}
+static inline int qp_is_bool(qp_types_t tp)
+{
+ return tp == QP_TRUE || tp == QP_FALSE;
+}
+static inline int qp_is_raw_term(qp_obj_t * qp_obj)
+{
+ return (qp_obj->tp == QP_RAW &&
+ qp_obj->len &&
+ (char) qp_obj->via.raw[qp_obj->len - 1] == '\0');
+}
+
+/* Add to packer functions */
+int qp_add_raw(qp_packer_t * packer, const unsigned char * raw, size_t len);
+int qp_add_string(qp_packer_t * packer, const char * str);
+int qp_add_string_term(qp_packer_t * packer, const char * str);
+int qp_add_string_term_n(qp_packer_t * packer, const char * str, size_t n);
+
+int qp_add_raw_term(qp_packer_t * packer, const unsigned char * raw, size_t len);
+int qp_add_double(qp_packer_t * packer, double real);
+int qp_add_int64(qp_packer_t * packer, int64_t integer);
+int qp_add_true(qp_packer_t * packer);
+int qp_add_false(qp_packer_t * packer);
+int qp_add_null(qp_packer_t * packer);
+int qp_add_type(qp_packer_t * packer, qp_types_t tp);
+int qp_add_fmt(qp_packer_t * packer, const char * fmt, ...);
+int qp_add_fmt_safe(qp_packer_t * packer, const char * fmt, ...);
+
+/* Add to file-packer functions */
+int qp_fadd_type(qp_fpacker_t * fpacker, qp_types_t tp);
+int qp_fadd_raw(qp_fpacker_t * fpacker, const unsigned char * raw, size_t len);
+int qp_fadd_string(qp_fpacker_t * fpacker, const char * str);
+int qp_fadd_int64(qp_fpacker_t * fpacker, int64_t integer);
+int qp_fadd_double(qp_fpacker_t * fpacker, double real);
+
+/* creates a valid qpack buffer of length 3 holding an int16 type. */
+#define QP_PACK_INT16(BUF__, N__) \
+unsigned char BUF__[3];\
+BUF__[0] = QP_INT16; \
+memcpy(&BUF__[1], &N__, 2);
+
+#endif /* QPACK_H_ */
--- /dev/null
+/*
+ * qpjson.h - Convert between QPack and JSON
+ */
+#ifndef QPJSON_H_
+#define QPJSON_H_
+
+#include <stddef.h>
+#include <yajl/yajl_gen.h>
+#include <yajl/yajl_parse.h>
+
+enum
+{
+ QPJSON_FLAG_BEAUTIFY =1<<0,
+ QPJSON_FLAG_VALIDATE_UTF8 =1<<1,
+};
+
+yajl_gen_status qpjson_qp_to_json(
+ void * src,
+ size_t src_n,
+ unsigned char ** dst,
+ size_t * dst_n,
+ int flags);
+
+yajl_status qpjson_json_to_qp(
+ const void * src,
+ size_t src_n,
+ char ** dst,
+ size_t * dst_n);
+
+#endif /* QPJSON_H_ */
--- /dev/null
+/*
+ * api.h - SiriDB HTTP API.
+ */
+#ifndef SIRI_API_H_
+#define SIRI_API_H_
+
+#include <lib/http_parser.h>
+#include <uv.h>
+#include <siri/db/db.h>
+#include <siri/service/request.h>
+
+typedef enum
+{
+ SIRI_API_CT_TEXT,
+ SIRI_API_CT_JSON,
+ SIRI_API_CT_QPACK,
+} siri_api_content_t;
+
+typedef enum
+{
+ SIRI_API_RT_NONE,
+ SIRI_API_RT_QUERY,
+ SIRI_API_RT_INSERT,
+ SIRI_APT_RT_SERVICE,
+} siri_api_req_t;
+
+typedef enum
+{
+ E200_OK,
+ E400_BAD_REQUEST,
+ E401_UNAUTHORIZED,
+ E403_FORBIDDEN,
+ E404_NOT_FOUND,
+ E405_METHOD_NOT_ALLOWED,
+ E415_UNSUPPORTED_MEDIA_TYPE,
+ E422_UNPROCESSABLE_ENTITY,
+ E500_INTERNAL_SERVER_ERROR,
+ E503_SERVICE_UNAVAILABLE
+} siri_api_header_t;
+
+typedef struct siri_api_request_s siri_api_request_t;
+
+typedef int (*on_state_cb_t)(siri_api_request_t * ar, const char * at, size_t n);
+
+int siri_api_init(void);
+int siri_api_send(
+ siri_api_request_t * ar,
+ siri_api_header_t ht,
+ unsigned char * src,
+ size_t n);
+
+struct siri_api_request_s
+{
+ uint32_t tp; /* maps to siridb_tee_t flags for cleanup */
+ uint32_t ref;
+ on_state_cb_t on_state;
+ siridb_t * siridb;
+ void * origin; /* can be a user, server or NULL */
+ char * buf;
+ size_t len;
+ size_t size;
+ uv_stream_t * stream;
+ siri_api_content_t content_type;
+ siri_api_req_t request_type;
+ service_request_t service_type;
+ _Bool service_authenticated;
+ http_parser parser;
+ uv_write_t req;
+};
+
+#endif /* SIRI_API_H_ */
--- /dev/null
+/*
+ * args.h - Parse SiriDB command line arguments.
+ */
+#ifndef SIRI_ARGS_H_
+#define SIRI_ARGS_H_
+
+typedef struct siri_args_s siri_args_t;
+
+#include <inttypes.h>
+#include <argparse/argparse.h>
+#include <siri/siri.h>
+
+/* arguments are configured and parsed here */
+void siri_args_parse(siri_t * siri, int argc, char *argv[]);
+
+struct siri_args_s
+{
+ /* true/false props */
+ int32_t version;
+ int32_t log_colorized;
+ int32_t managed;
+
+ /* string props */
+ char config[ARGPARSE_MAX_LEN_ARG];
+ char log_level[ARGPARSE_MAX_LEN_ARG];
+};
+
+#endif /* SIRI_ARGS_H_ */
--- /dev/null
+/*
+ * async.h - SiriDB async wrapper.
+ */
+#ifndef SIRI_ASYC_H_
+#define SIRI_ASYC_H_
+
+typedef struct siri_async_s siri_async_t;
+
+#include <uv.h>
+#include <inttypes.h>
+
+void siri_async_close(uv_handle_t * handle);
+void siri_async_decref(uv_async_t ** handle);
+
+#define siri_async_incref(HANDLE__) ((siri_async_t *) HANDLE__->data)->ref++
+
+struct siri_async_s
+{
+ uv_close_cb free_cb;
+ uint8_t ref;
+};
+
+#endif /* SIRI_ASYC_H_ */
--- /dev/null
+/*
+ * backup.h - Set SiriDB in backup mode.
+ */
+#ifndef SIRI_BACKUP_H_
+#define SIRI_BACKUP_H_
+
+#include <siri/siri.h>
+#include <siri/db/db.h>
+
+int siri_backup_init(siri_t * siri);
+void siri_backup_destroy(siri_t * siri);
+int siri_backup_enable(siri_t * siri, siridb_t * siridb);
+int siri_backup_disable(siri_t * siri, siridb_t * siridb);
+
+
+#endif /* SIRI_BACKUP_H_ */
--- /dev/null
+/*
+ * buffersync.h - Buffer sync.
+ */
+#ifndef SIRI_BUFFERSYNC_H_
+#define SIRI_BUFFERSYNC_H_
+
+#include <siri/siri.h>
+
+void siri_buffersync_init(siri_t * siri);
+void siri_buffersync_stop(siri_t * siri);
+
+#endif /* SIRI_HEARTBEAT_H_ */
--- /dev/null
+/*
+ * cfg.h - Read the global SiriDB configuration file. (usually siridb.conf)
+ */
+#ifndef SIRI_CFG_H_
+#define SIRI_CFG_H_
+
+typedef struct siri_cfg_s siri_cfg_t;
+
+#define SIRI_CFG_MAX_LEN_ADDRESS 256
+
+/* do not use more than x percent for the max limit for open sharding files */
+#define RLIMIT_PERC_FOR_SHARDING 0.5
+
+#define MAX_OPEN_FILES_LIMIT 32768
+#define MIN_OPEN_FILES_LIMIT 3
+#define DEFAULT_OPEN_FILES_LIMIT MAX_OPEN_FILES_LIMIT
+
+#include <inttypes.h>
+#include <limits.h>
+#include <siri/siri.h>
+#include <xpath/xpath.h>
+
+void siri_cfg_init(siri_t * siri);
+void siri_cfg_destroy(siri_t * siri);
+
+struct siri_cfg_s
+{
+ uint32_t optimize_interval;
+ uint32_t buffer_sync_interval;
+
+ uint16_t listen_client_port;
+ uint16_t listen_backend_port;
+ uint16_t heartbeat_interval;
+ uint16_t max_open_files;
+
+ uint16_t http_status_port;
+ uint16_t http_api_port;
+ uint8_t pipe_support;
+ uint8_t ip_support;
+ uint8_t shard_compression;
+ uint8_t shard_auto_duration;
+
+ char * bind_client_addr;
+ char * bind_backend_addr;
+ char server_address[SIRI_CFG_MAX_LEN_ADDRESS];
+ char db_path[XPATH_MAX];
+ char pipe_client_name[XPATH_MAX];
+};
+
+#endif /* SIRI_CFG_H_ */
--- /dev/null
+/*
+ * access.h - Access constants and functions.
+ */
+#ifndef SIRIDB_ACCESS_H_
+#define SIRIDB_ACCESS_H_
+
+/* definitions of all access rights */
+#define SIRIDB_ACCESS_SHOW 1
+#define SIRIDB_ACCESS_COUNT 2
+#define SIRIDB_ACCESS_LIST 4
+#define SIRIDB_ACCESS_SELECT 8
+#define SIRIDB_ACCESS_INSERT 16
+#define SIRIDB_ACCESS_CREATE 32
+#define SIRIDB_ACCESS_ALTER 64
+#define SIRIDB_ACCESS_DROP 128
+#define SIRIDB_ACCESS_GRANT 256
+#define SIRIDB_ACCESS_REVOKE 512
+
+/* this is a save size since access string cannot contain double items */
+#define SIRIDB_ACCESS_STR_MAX 128
+
+/* profile definitions */
+#define SIRIDB_ACCESS_PROFILE_READ \
+ SIRIDB_ACCESS_SHOW | \
+ SIRIDB_ACCESS_COUNT | \
+ SIRIDB_ACCESS_LIST | \
+ SIRIDB_ACCESS_SELECT
+
+#define SIRIDB_ACCESS_PROFILE_WRITE \
+ SIRIDB_ACCESS_PROFILE_READ | \
+ SIRIDB_ACCESS_INSERT | \
+ SIRIDB_ACCESS_CREATE
+
+#define SIRIDB_ACCESS_PROFILE_MODIFY \
+ SIRIDB_ACCESS_PROFILE_WRITE | \
+ SIRIDB_ACCESS_ALTER | \
+ SIRIDB_ACCESS_DROP
+
+#define SIRIDB_ACCESS_PROFILE_FULL \
+ SIRIDB_ACCESS_PROFILE_MODIFY | \
+ SIRIDB_ACCESS_GRANT | \
+ SIRIDB_ACCESS_REVOKE
+
+typedef struct siridb_access_repr_s siridb_access_repr_t;
+
+#include <cleri/cleri.h>
+
+uint32_t siridb_access_from_strn(const char * str, size_t n);
+
+/* children must be children from a cleri_list where values are comma
+ * separated. The children node must be valid and contain at least one access
+ * string.
+ */
+uint32_t siridb_access_from_children(cleri_children_t * children);
+
+void siridb_access_to_str(char * str, uint32_t access_bit);
+
+struct siridb_access_repr_s
+{
+ const char * repr;
+ uint32_t access_bit;
+};
+
+#endif /* SIRIDB_ACCESS_H_ */
--- /dev/null
+/*
+ * aggregate.h - SiriDB aggregation methods.
+ */
+#ifndef SIRIDB_AGGREGATE_H_
+#define SIRIDB_AGGREGATE_H_
+
+typedef struct siridb_aggr_s siridb_aggr_t;
+
+#include <siri/db/points.h>
+#include <siri/grammar/gramp.h>
+#include <vec/vec.h>
+#include <cexpr/cexpr.h>
+#include <qpack/qpack.h>
+#include <pcre2.h>
+
+siridb_points_t * siridb_aggregate_run(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+void siridb_init_aggregates(void);
+vec_t * siridb_aggregate_list(cleri_children_t * children, char * err_msg);
+void siridb_aggregate_list_free(vec_t * alist);
+int siridb_aggregate_can_skip(cleri_children_t * children);
+
+struct siridb_aggr_s
+{
+ uint32_t gid;
+ cexpr_operator_t filter_opr;
+ uint8_t filter_tp;
+ uint64_t group_by;
+ uint64_t limit;
+ uint64_t offset;
+ double timespan; /* used for derivative */
+ pcre2_code * regex; \
+ pcre2_match_data * match_data;
+ qp_via_t filter_via;
+};
+
+#endif /* SIRIDB_AGGREGATE_H_ */
--- /dev/null
+/*
+ * auth.h - Handle SiriDB authentication.
+ */
+#ifndef SIRIDB_AUTH_H_
+#define SIRIDB_AUTH_H_
+
+#include <stddef.h>
+#include <siri/net/clserver.h>
+#include <qpack/qpack.h>
+#include <siri/net/protocol.h>
+
+cproto_server_t siridb_auth_user_request(
+ sirinet_stream_t * client,
+ qp_obj_t * qp_username,
+ qp_obj_t * qp_password,
+ qp_obj_t * qp_dbname);
+
+bproto_server_t siridb_auth_server_request(
+ sirinet_stream_t * client,
+ qp_obj_t * qp_uuid,
+ qp_obj_t * qp_dbname,
+ qp_obj_t * qp_version,
+ qp_obj_t * qp_min_version);
+
+#endif /* SIRIDB_AUTH_H_ */
--- /dev/null
+/*
+ * buffer.h - Buffer for integer and double values.
+ */
+#ifndef SIRIDB_BUFFER_H_
+#define SIRIDB_BUFFER_H_
+
+typedef struct siridb_buffer_s siridb_buffer_t;
+
+#include <siri/db/db.h>
+#include <siri/db/series.h>
+#include <siri/db/points.h>
+#include <unistd.h>
+
+#define MAX_BUFFER_SZ 1048576
+
+siridb_buffer_t * siridb_buffer_new(void);
+void siridb_buffer_free(siridb_buffer_t * buffer);
+void siridb_buffer_close(siridb_buffer_t * buffer);
+_Bool siridb_buffer_is_valid_size(ssize_t ssize);
+void siridb_buffer_set_path(siridb_buffer_t * buffer, const char * str);
+int siridb_buffer_new_series(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series);
+int siridb_buffer_open(siridb_buffer_t * buffer);
+int siridb_buffer_load(siridb_t * siridb);
+int siridb_buffer_test_path(siridb_t * siridb);
+int siridb_buffer_write_empty(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series);
+int siridb_buffer_write_point(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series,
+ uint64_t * ts,
+ qp_via_t * val);
+
+struct siridb_buffer_s
+{
+ size_t size; /* size for one series inside the buffer */
+ size_t _to_size; /* optional new size from database.conf */
+ size_t len; /* number of points allocated per series */
+ char * template; /* template for writing an empty buffer */
+ char * path; /* path where the buffer file is stored */
+ vec_t * empty; /* list with empty buffer spaces */
+ FILE * fp; /* buffer file pointer */
+ int fd; /* buffer file descriptor */
+};
+
+static inline int siridb_buffer_fsync(siridb_buffer_t * buffer)
+{
+ return (buffer->fp == NULL) ? 0 : fsync(buffer->fd);
+}
+
+#endif /* SIRIDB_BUFFER_H_ */
--- /dev/null
+/*
+ * db.h - SiriDB database.
+ */
+#ifndef SIRIDB_H_
+#define SIRIDB_H_
+
+typedef struct siridb_s siridb_t;
+
+#define SIRIDB_MAX_SIZE_ERR_MSG 1024
+#define SIRIDB_MAX_DBNAME_LEN 256 /* 255 + NULL */
+#define SIRIDB_SCHEMA 6
+#define SIRIDB_FLAG_REINDEXING 1
+#define SIRIDB_FLAG_DROPPED 2
+
+#define DEF_DROP_THRESHOLD 1.0 /* 100% */
+#define DEF_SELECT_POINTS_LIMIT 1000000 /* one million */
+#define DEF_LIST_LIMIT 10000 /* ten thousand */
+
+#include <string.h>
+#include <uv.h>
+#include <qpack/qpack.h>
+#include <ctree/ctree.h>
+#include <imap/imap.h>
+#include <imap/imap.h>
+#include <iso8601/iso8601.h>
+#include <llist/llist.h>
+#include <siri/err.h>
+#include <siri/db/time.h>
+#include <siri/db/user.h>
+#include <siri/db/server.h>
+#include <siri/db/pools.h>
+#include <siri/db/fifo.h>
+#include <siri/db/replicate.h>
+#include <siri/db/reindex.h>
+#include <siri/db/groups.h>
+#include <siri/db/tasks.h>
+#include <siri/db/time.h>
+#include <siri/db/buffer.h>
+#include <siri/db/tee.h>
+#include <siri/db/tags.h>
+
+
+int32_t siridb_get_uptime(siridb_t * siridb);
+int8_t siridb_get_idle_percentage(siridb_t * siridb);
+int siridb_is_db_path(const char * dbpath);
+siridb_t * siridb_new(const char * dbpath, int lock_flags);
+siridb_t * siridb_get(llist_t * siridb_list, const char * dbname);
+siridb_t * siridb_getn(llist_t * siridb_list, const char * dbname, size_t n);
+siridb_t * siridb_get_by_qp(llist_t * siridb_list, qp_obj_t * qp_dbname);
+int siridb_decref_cb(siridb_t * siridb, void * args);
+ssize_t siridb_get_file(char ** buffer, siridb_t * siridb);
+int siridb_open_files(siridb_t * siridb);
+int siridb_save(siridb_t * siridb);
+void siridb__free(siridb_t * siridb);
+void siridb_drop(siridb_t * siridb);
+void siridb_update_shard_expiration(siridb_t * siridb);
+
+#define siridb_incref(siridb__) __atomic_add_fetch(&(siridb__)->ref, 1, __ATOMIC_SEQ_CST)
+#define siridb_decref(siridb__) \
+ if (!__atomic_sub_fetch(&(siridb__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__free(siridb__)
+#define siridb_is_reindexing(siridb) (siridb->flags & SIRIDB_FLAG_REINDEXING)
+
+struct siridb_s
+{
+ uint16_t ref;
+ uint8_t flags;
+ uint8_t pad0;
+ uint32_t max_series_id;
+ uint16_t insert_tasks;
+ uint16_t shard_mask_num;
+ uint16_t shard_mask_log;
+ uint32_t select_points_limit;
+ uint32_t list_limit;
+ uuid_t uuid;
+ iso8601_tz_t tz;
+ struct timespec start_time; /* to calculate up-time. */
+ uint64_t duration_num; /* number duration in s, ms, us or ns */
+ uint64_t duration_log; /* log duration in s, ms, us or ns */
+ uint64_t exp_at_num; /* UNIX time stamp in s, ms, us or ns */
+ uint64_t exp_at_log; /* UNIX time stamp in s, ms, us or ns */
+ uint64_t expiration_num; /* number duration in s, ms, us or ns */
+ uint64_t expiration_log; /* log duration in s, ms, us or ns */
+ char * dbname;
+ char * dbpath;
+ double drop_threshold;
+ size_t received_points;
+ size_t selected_points;
+
+ siridb_time_t * time;
+ siridb_server_t * server;
+ siridb_server_t * replica;
+ llist_t * users;
+ llist_t * servers;
+ siridb_pools_t * pools;
+ ct_t * series;
+ imap_t * series_map;
+ uv_mutex_t series_mutex;
+ uv_mutex_t shards_mutex;
+ uv_mutex_t values_mutex;
+ imap_t * shards; /* contains lists with shards */
+ FILE * dropped_fp;
+ qp_fpacker_t * store;
+ siridb_fifo_t * fifo;
+ siridb_replicate_t * replicate;
+ siridb_reindex_t * reindex;
+ siridb_groups_t * groups;
+ siridb_tags_t * tags;
+ siridb_buffer_t * buffer;
+ siridb_tee_t * tee;
+ siridb_tasks_t tasks;
+};
+
+#endif /* SIRIDB_H_ */
--- /dev/null
+/*
+ * ffile.h - FIFO file.
+ */
+#ifndef SIRIDB_FFILE_H_
+#define SIRIDB_FFILE_H_
+
+typedef enum
+{
+ FFILE_NO_FREE_SPACE=-2,
+ FFILE_ERROR,
+ FFILE_SUCCESS
+} siridb_ffile_result_t;
+
+typedef struct siridb_ffile_s siridb_ffile_t;
+
+#include <siri/net/pkg.h>
+
+void siridb_ffile_open(siridb_ffile_t * ffile, const char * opentype);
+siridb_ffile_t * siridb_ffile_new(
+ uint64_t id,
+ const char * path,
+ sirinet_pkg_t * pkg);
+int siridb_ffile_check_fn(const char * fn);
+void siridb_ffile_free(siridb_ffile_t * ffile);
+void siridb_ffile_unlink(siridb_ffile_t * ffile);
+sirinet_pkg_t * siridb_ffile_pop(siridb_ffile_t * ffile);
+int siridb_ffile_pop_commit(siridb_ffile_t * ffile);
+siridb_ffile_result_t siridb_ffile_append(
+ siridb_ffile_t * ffile,
+ sirinet_pkg_t * pkg);
+
+struct siridb_ffile_s
+{
+ uint64_t id;
+ char * fn;
+ uint32_t free_space;
+ uint32_t next_size; /* must be uint32_t (4 bytes) */
+ FILE * fp;
+ int fd;
+ long int size;
+};
+
+#endif /* SIRIDB_FFILE_H_ */
--- /dev/null
+/*
+ * fifo.h - First in, first out file buffer.
+ */
+#ifndef SIRIDB_FIFO_H_
+#define SIRIDB_FIFO_H_
+
+typedef struct siridb_fifo_s siridb_fifo_t;
+
+#include <stddef.h>
+#include <siri/db/db.h>
+#include <llist/llist.h>
+#include <siri/db/ffile.h>
+
+
+siridb_fifo_t * siridb_fifo_new(siridb_t * siridb);
+void siridb_fifo_free(siridb_fifo_t * fifo);
+size_t siridb_fifo_size(siridb_fifo_t * fifo);
+int siridb_fifo_append(siridb_fifo_t * fifo, sirinet_pkg_t * pkg);
+sirinet_pkg_t * siridb_fifo_pop(siridb_fifo_t * fifo);
+int siridb_fifo_commit(siridb_fifo_t * fifo);
+int siridb_fifo_commit_err(siridb_fifo_t * fifo);
+int siridb_fifo_close(siridb_fifo_t * fifo);
+int siridb_fifo_open(siridb_fifo_t * fifo);
+
+/*
+ * Value is greater than 0 when the fifo has data or 0 when empty.
+ * Use this to check if the fifo has data. (must be done before calling pop)
+ */
+#define siridb_fifo_has_data(fifo) fifo->out->next_size
+
+
+/*
+ * Returns 1 if the fifo buffer is open or 0 if closed.
+ */
+#define siridb_fifo_is_open(fifo) (fifo->in->fp != NULL)
+
+struct siridb_fifo_s
+{
+ char * path;
+ llist_t * fifos;
+ siridb_ffile_t * in;
+ siridb_ffile_t * out;
+ ssize_t max_id; /* max_id can be -1 */
+};
+
+#endif /* SIRIDB_FIFO_H_ */
--- /dev/null
+/*
+ * forward.h - Handle forwarding series while re-indexing.
+ */
+#ifndef SIRIDB_FORWARD_H_
+#define SIRIDB_FORWARD_H_
+
+typedef struct siridb_forward_s siridb_forward_t;
+
+#include <inttypes.h>
+#include <uv.h>
+#include <siri/db/db.h>
+
+siridb_forward_t * siridb_forward_new(siridb_t * siridb);
+void siridb_forward_free(siridb_forward_t * forward);
+void siridb_forward_points_to_pools(uv_async_t * handle);
+
+struct siridb_forward_s
+{
+ uv_close_cb free_cb; /* must be on top */
+ uint8_t ref;
+ uint16_t size; /* number of packers (one for each pool - 1) */
+ siridb_t * siridb;
+ qp_packer_t * packer[];
+};
+
+#endif /* SIRIDB_FORWARD_H_ */
--- /dev/null
+/*
+ * group.h - Group (saved regular expressions).
+ */
+#ifndef SIRIDB_GROUP_H_
+#define SIRIDB_GROUP_H_
+
+#define PCRE2_CODE_UNIT_WIDTH 8
+
+enum
+{
+ GROUP_FLAG_INIT = 1<<0,
+ GROUP_FLAG_DROPPED = 1<<1,
+};
+
+typedef struct siridb_group_s siridb_group_t;
+
+#include <vec/vec.h>
+#include <siri/db/series.h>
+#include <siri/db/db.h>
+#include <pcre2.h>
+
+siridb_group_t * siridb_group_new(
+ const char * source,
+ size_t source_len,
+ char * err_msg);
+int siridb_group_set_name(
+ siridb_t * siridb,
+ siridb_group_t * group,
+ const char * name,
+ char * err_msg);
+int siridb_group_update_expression(
+ siridb_groups_t * groups,
+ siridb_group_t * group,
+ const char * source,
+ size_t source_len,
+ char * err_msg);
+void siridb_group_cleanup(siridb_group_t * group);
+int siridb_group_test_series(siridb_group_t * group, siridb_series_t * series);
+int siridb_group_cexpr_cb(siridb_group_t * group, cexpr_condition_t * cond);
+void siridb_group_prop(siridb_group_t * group, qp_packer_t * packer, int prop);
+int siridb_group_is_remote_prop(uint32_t prop);
+void siridb__group_decref(siridb_group_t * group);
+void siridb__group_free(siridb_group_t * group);
+
+#define siridb_group_incref(group__) __atomic_add_fetch(&(group__)->ref, 1, __ATOMIC_SEQ_CST)
+#define siridb_group_decref(group__) \
+ if (!__atomic_sub_fetch(&(group__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__group_free(group__)
+
+struct siridb_group_s
+{
+ uint16_t ref;
+ uint16_t flags;
+ uint32_t n; /* total series (needs an update from all pools) */
+ char * name;
+ char * source; /* pattern/flags representation */
+ vec_t * series;
+ pcre2_code * regex;
+ pcre2_match_data * match_data;
+};
+
+#endif /* SIRIDB_GROUP_H_ */
--- /dev/null
+/*
+ * groups.h - Groups (saved regular expressions).
+ *
+ * Info groups->mutex:
+ *
+ * Main thread:
+ * groups->groups : read (no lock) write (lock)
+ * groups->nseries : read (lock) write (lock)
+ * groups->ngroups : read (lock) write (lock)
+ * group->series : read (lock) write (not allowed)
+
+ * Other threads:
+ * groups->groups : read (lock) write (not allowed)
+ * groups->nseries : read (lock) write (lock)
+ * groups->ngroups : read (lock) write (lock)
+ *
+ * Group thread:
+ * group->series : read (no lock) write (lock)
+ *
+ * Note: One exception to 'not allowed' are the free functions
+ * since they only run when no other references to the object exist.
+ */
+#ifndef SIRIDB_GROUPS_H_
+#define SIRIDB_GROUPS_H_
+
+typedef enum
+{
+ GROUPS_RUNNING,
+ GROUPS_STOPPING,
+ GROUPS_CLOSED
+} siridb_groups_status_t;
+
+typedef struct siridb_groups_s siridb_groups_t;
+
+enum
+{
+ GROUPS_FLAG_DROPPED_SERIES = 1<<0,
+};
+
+#include <ctree/ctree.h>
+#include <vec/vec.h>
+#include <uv.h>
+#include <siri/db/db.h>
+#include <siri/net/pkg.h>
+
+int siridb_groups_init(siridb_t * siridb);
+void siridb_groups_start(siridb_t * siridb);
+int siridb_groups_save(siridb_groups_t * groups);
+ssize_t siridb_groups_get_file(char ** buffer, siridb_t * siridb);
+void siridb_groups_init_nseries(siridb_groups_t * groups);
+sirinet_pkg_t * siridb_groups_pkg(siridb_groups_t * groups, uint16_t pid);
+int siridb_groups_drop_group(
+ siridb_groups_t * groups,
+ const char * name,
+ char * err_msg);
+void siridb_groups_destroy(siridb_groups_t * groups);
+void siridb_groups_incref(siridb_groups_t * groups);
+void siridb_groups_decref(siridb_groups_t * groups);
+int siridb_groups_add_group(
+ siridb_t * siridb,
+ const char * name,
+ const char * source,
+ size_t source_len,
+ char * err_msg);
+int siridb_groups_add_series(
+ siridb_groups_t * groups,
+ siridb_series_t * series);
+void siridb__groups_free(siridb_groups_t * groups);
+
+struct siridb_groups_s
+{
+ uint8_t status;
+ uint8_t flags;
+ uint8_t ref;
+ char * fn;
+ ct_t * groups;
+ vec_t * nseries; /* list of series we need to assign to groups */
+ vec_t * ngroups; /* list of groups which need initialization */
+ uv_mutex_t mutex;
+ uv_thread_t thread;
+};
+
+#define siridb_groups_incref(groups__) __atomic_add_fetch(&(groups__)->ref, 1, __ATOMIC_SEQ_CST)
+#define siridb_groups_decref(groups__) \
+ if (!__atomic_sub_fetch(&(groups__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__groups_free(groups__)
+
+
+#endif /* SIRIDB_GROUPS_H_ */
--- /dev/null
+/*
+ * initsync.h - Initial replica synchronization.
+ */
+#ifndef SIRIDB_INITSYNC_H_
+#define SIRIDB_INITSYNC_H_
+
+typedef struct siridb_initsync_s siridb_initsync_t;
+
+#include <stdio.h>
+#include <uv.h>
+#include <inttypes.h>
+#include <siri/db/db.h>
+#include <siri/db/series.h>
+#include <siri/net/pkg.h>
+
+siridb_initsync_t * siridb_initsync_open(siridb_t * siridb, int create_new);
+void siridb_initsync_free(siridb_initsync_t ** initsync);
+void siridb_initsync_run(uv_timer_t * timer);
+void siridb_initsync_fopen(siridb_initsync_t * initsync, const char * opentype);
+const char * siridb_initsync_sync_progress(siridb_t * siridb);
+
+struct siridb_initsync_s
+{
+ FILE * fp;
+ char * fn;
+ int fd;
+ long int size;
+ uint32_t * next_series_id;
+ sirinet_pkg_t * pkg_points;
+ sirinet_pkg_t * pkg_tags;
+};
+
+#endif /* SIRIDB_INITSYNC_H_ */
--- /dev/null
+/*
+ * insert.h - Handler database inserts.
+ */
+#ifndef SIRIDB_INSERT_H_
+#define SIRIDB_INSERT_H_
+
+#define INSERT_FLAG_TEST 1
+#define INSERT_FLAG_TESTED 2
+#define INSERT_FLAG_POOL 4
+#define INSERT_FLAG_INIT_REPL 8
+
+typedef enum
+{
+ ERR_EXPECTING_ARRAY=-10,
+ ERR_EXPECTING_SERIES_NAME,
+ ERR_EXPECTING_MAP_OR_ARRAY,
+ ERR_EXPECTING_INTEGER_TS,
+ ERR_TIMESTAMP_OUT_OF_RANGE,
+ ERR_UNSUPPORTED_VALUE,
+ ERR_EXPECTING_AT_LEAST_ONE_POINT,
+ ERR_EXPECTING_NAME_AND_POINTS,
+ ERR_INCOMPATIBLE_SERVER_VERSION,
+ ERR_MEM_ALLOC, /* This is a critical error. */
+} siridb_insert_err_t;
+
+enum
+{
+ INSERT_LOCAL_CANCELLED=-2,
+ INSERT_LOCAL_ERROR,
+ INSERT_LOCAL_SUCESS,
+};
+
+typedef struct siridb_insert_s siridb_insert_t;
+typedef struct siridb_insert_local_s siridb_insert_local_t;
+
+#include <siri/db/db.h>
+#include <qpack/qpack.h>
+#include <siri/db/forward.h>
+#include <uv.h>
+#include <siri/db/pcache.h>
+
+ssize_t siridb_insert_assign_pools(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_packer_t * packer[]);
+const char * siridb_insert_err_msg(siridb_insert_err_t err);
+siridb_insert_t * siridb_insert_new(
+ siridb_t * siridb,
+ uint16_t pid,
+ sirinet_stream_t * client);
+void siridb_insert_free(siridb_insert_t * insert);
+int siridb_insert_points_to_pools(siridb_insert_t * insert, size_t npoints);
+int insert_init_backend_local(
+ siridb_t * siridb,
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg,
+ uint8_t flags);
+
+struct siridb_insert_s
+{
+ uv_close_cb free_cb; /* must be on top */
+ uint8_t ref;
+ uint8_t flags;
+ uint16_t pid;
+ sirinet_stream_t * client;
+ size_t npoints; /* number of points */
+ uint16_t packer_size; /* number of packers (one for each pool) */
+ qp_packer_t * packer[];
+};
+
+struct siridb_insert_local_s
+{
+ uv_close_cb free_cb; /* must be on top */
+ uint8_t ref;
+ uint8_t flags;
+ int8_t status;
+ qp_unpacker_t unpacker;
+ qp_obj_t qp_series_name;
+ siridb_t * siridb;
+ sirinet_promise_t * promise;
+ siridb_forward_t * forward;
+ siridb_pcache_t * pcache;
+};
+
+#endif /* SIRIDB_INSERT_H_ */
--- /dev/null
+/*
+ * listener.h - Contains functions for processing queries.
+ */
+#ifndef SIRIDB_LISTENER_H_
+#define SIRIDB_LISTENER_H_
+
+#include <uv.h>
+#include <siri/grammar/grammar.h>
+
+void siridb_init_listener(void);
+
+uv_async_cb siridb_node_get_enter(enum cleri_grammar_ids gid);
+uv_async_cb siridb_node_get_exit(enum cleri_grammar_ids gid);
+
+#endif /* SIRIDB_LISTENER_H_ */
--- /dev/null
+/*
+ * lookup.h - Find and assign to which pool series belong.
+ */
+#ifndef SIRIDB_LOOKUP_H_
+#define SIRIDB_LOOKUP_H_
+
+#include <inttypes.h>
+#include <stddef.h>
+
+#define SIRIDB_LOOKUP_SZ 8192
+
+typedef uint_fast16_t siridb_lookup_t[SIRIDB_LOOKUP_SZ];
+
+uint16_t siridb_lookup_sn(siridb_lookup_t * lookup, const char * sn);
+uint16_t siridb_lookup_sn_raw(
+ siridb_lookup_t * lookup,
+ const char * sn,
+ size_t len);
+siridb_lookup_t * siridb_lookup_new(uint_fast16_t num_pools);
+void siridb_lookup_free(siridb_lookup_t * lookup);
+
+#endif /* SIRIDB_LOOKUP_H_ */
--- /dev/null
+/*
+ * median.h - Calculate median, median high and median low.
+ */
+#ifndef SIRIDB_MEDIAN_H_
+#define SIRIDB_MEDIAN_H_
+
+#include <siri/db/points.h>
+#include <inttypes.h>
+
+struct siridb_point_s;
+struct siridb_points_s;
+
+/* sets point int64 or real depending on the given points */
+int siridb_median_find_n(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ uint64_t n);
+
+/* sets points real, even when the given points are integer type */
+int siridb_median_real(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ double percentage);
+
+
+#endif /* SIRIDB_MEDIAN_H_ */
--- /dev/null
+/*
+ * misc.h - Miscellaneous functions used by SiriDB.
+ */
+#ifndef SIRIDB_MISC_H_
+#define SIRIDB_MISC_H_
+
+#include <inttypes.h>
+#include <qpack/qpack.h>
+
+#define siridb_misc_get_fn(__fn, __path, __filename) \
+ char __fn[strlen(__path) + strlen(__filename) + 1]; \
+ sprintf(__fn, "%s%s", __path, __filename);
+
+/* Schema File Check
+ * Needs fn and unpacker, free unpacker in case of an error.
+ * Returns current function with -1 in case of an error and
+ * raises a SIGNAL in case of a memory error.
+ */
+#define siridb_misc_schema_check(__schema) \
+ /* read and check schema */ \
+ qp_obj_t qp_schema; \
+ if (!qp_is_array(qp_next(unpacker, NULL)) || \
+ qp_next(unpacker, &qp_schema) != QP_INT64 || \
+ qp_schema.via.int64 != __schema) \
+ { \
+ log_critical("Invalid schema detected in '%s'", fn); \
+ qp_unpacker_ff_free(unpacker); \
+ return -1; \
+ }
+
+qp_unpacker_t * siridb_misc_open_schema_file(uint8_t schema, const char * fn);
+
+
+#endif /* SIRIDB_MISC_H_ */
--- /dev/null
+/*
+ * nodes.h - Contains logic for cleri nodes which we need to parse.
+ */
+#ifndef SIRIDB_NODES_H_
+#define SIRIDB_NODES_H_
+
+typedef struct siridb_nodes_s siridb_nodes_t;
+
+#include <cleri/cleri.h>
+#include <siri/db/db.h>
+#include <uv.h>
+
+void siridb_nodes_free(siridb_nodes_t * nodes);
+void siridb_nodes_next(siridb_nodes_t ** nodes);
+
+struct siridb_nodes_s
+{
+ cleri_node_t * node;
+ uv_async_cb cb;
+ struct siridb_nodes_s * next;
+};
+
+#endif /* SIRIDB_NODES_H_ */
--- /dev/null
+/*
+ * pcache.h - Points structure with notion of its size.
+ */
+#ifndef SIRIDB_PCACHE_H_
+#define SIRIDB_PCACHE_H_
+
+typedef struct siridb_pcache_s siridb_pcache_t;
+
+#include <siri/db/points.h>
+
+siridb_pcache_t * siridb_pcache_new(points_tp tp);
+int siridb_pcache_add_point(
+ siridb_pcache_t * pcache,
+ uint64_t * ts,
+ qp_via_t * val);
+int siridb_pcache_add_ts_obj(
+ siridb_pcache_t * pcache,
+ uint64_t * ts,
+ qp_obj_t * obj);
+
+#define siridb_pcache_free(pcache) \
+ siridb_points_free((siridb_points_t *) pcache)
+
+struct siridb_pcache_s
+{
+ size_t len;
+ points_tp tp;
+ siridb_point_t * data;
+ size_t size; /* addition to normal points type */
+};
+#endif /* SIRIDB_PCACHE_H_ */
--- /dev/null
+/*
+ * points.h - Array object for points.
+ */
+#ifndef SIRIDB_POINTS_H_
+#define SIRIDB_POINTS_H_
+
+#define POINTS_ZIP_THRESHOLD 5
+
+typedef enum
+{
+ TP_INT,
+ TP_DOUBLE,
+ TP_STRING
+} points_tp;
+
+typedef struct siridb_point_s siridb_point_t;
+typedef struct siridb_points_s siridb_points_t;
+
+#include <stdlib.h>
+#include <inttypes.h>
+#include <qpack/qpack.h>
+#include <vec/vec.h>
+
+void siridb_points_init(void);
+siridb_points_t * siridb_points_new(size_t size, points_tp tp);
+void siridb_points_free(siridb_points_t * points);
+int siridb_points_resize(siridb_points_t * points, size_t n);
+void siridb_points_add_point(
+ siridb_points_t *__restrict points,
+ uint64_t * ts,
+ qp_via_t * val);
+siridb_points_t * siridb_points_copy(siridb_points_t * points);
+int siridb_points_pack(siridb_points_t * points, qp_packer_t * packer);
+void siridb_points_ts_correction(siridb_points_t * points, double factor);
+int siridb_points_raw_pack(siridb_points_t * points, qp_packer_t * packer);
+siridb_points_t * siridb_points_merge(vec_t * plist, char * err_msg);
+unsigned char * siridb_points_zip_double(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size);
+unsigned char * siridb_points_zip_int(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size);
+unsigned char * siridb_points_zip_string(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size);
+unsigned char * siridb_points_raw_string(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size);
+void siridb_points_unzip_int(
+ siridb_points_t * points,
+ unsigned char * bits,
+ uint16_t len,
+ uint16_t cinfo,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+void siridb_points_unzip_double(
+ siridb_points_t * points,
+ unsigned char * bits,
+ uint16_t len,
+ uint16_t cinfo,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_points_unzip_string(
+ siridb_points_t * points,
+ uint8_t * bits,
+ uint16_t len,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_points_unzip_string_raw(
+ siridb_points_t * points,
+ uint8_t * bits,
+ uint16_t len);
+size_t siridb_points_get_size_zipped(uint16_t cinfo, uint16_t len);
+uint64_t siridb_points_get_interval(siridb_points_t * points);
+
+#define siridb_points_zip(p__, s__, e__, c__, z__) \
+((p__)->tp == TP_INT) ? \
+siridb_points_zip_int(p__, s__, e__, c__, z__) : \
+((p__)->tp == TP_DOUBLE) ? \
+siridb_points_zip_double(p__, s__, e__, c__, z__) : \
+siridb_points_zip_string(p__, s__, e__, c__, z__)
+
+struct siridb_point_s
+{
+ uint64_t ts;
+ qp_via_t val;
+};
+
+struct siridb_points_s
+{
+ size_t len;
+ points_tp tp;
+ siridb_point_t * data;
+};
+
+static inline size_t siridb_points_get_size_log(size_t cinfo)
+{
+ return cinfo & 0x8000 ? (cinfo ^ 0x8000) << 10 : cinfo;
+}
+
+
+#endif /* SIRIDB_POINTS_H_ */
--- /dev/null
+/*
+ * pool.h - SiriDB pool containing one or two servers.
+ */
+#ifndef SIRIDB_POOL_H_
+#define SIRIDB_POOL_H_
+
+typedef struct siridb_pool_s siridb_pool_t;
+typedef struct siridb_pool_walker_s siridb_pool_walker_t;
+
+#include <cexpr/cexpr.h>
+#include <inttypes.h>
+#include <siri/db/db.h>
+#include <siri/db/pools.h>
+#include <siri/db/server.h>
+#include <siri/net/pkg.h>
+#include <siri/net/promise.h>
+
+int siridb_pool_cexpr_cb(siridb_pool_walker_t * wpool, cexpr_condition_t * cond);
+int siridb_pool_online(siridb_pool_t * pool);
+int siridb_pool_available(siridb_pool_t * pool);
+int siridb_pool_accessible(siridb_pool_t * pool);
+int siridb_pool_send_pkg(
+ siridb_pool_t * pool,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promise_cb cb,
+ void * data,
+ int flags);
+void siridb_pool_add_server(siridb_pool_t * pool, siridb_server_t * server);
+
+
+struct siridb_pool_s
+{
+ uint16_t len;
+ siridb_server_t * server[2];
+};
+
+struct siridb_pool_walker_s
+{
+ uint_fast16_t pool;
+ uint_fast8_t servers;
+ size_t series;
+};
+
+#endif /* SIRIDB_POOL_H_ */
--- /dev/null
+/*
+ * pools.h - Collection of pools.
+ */
+#ifndef SIRIDB_POOLS_H_
+#define SIRIDB_POOLS_H_
+
+typedef struct siridb_pools_s siridb_pools_t;
+
+#include <inttypes.h>
+#include <siri/db/db.h>
+#include <siri/db/pool.h>
+#include <siri/db/server.h>
+#include <siri/net/pkg.h>
+#include <siri/net/promise.h>
+#include <siri/net/promises.h>
+#include <vec/vec.h>
+#include <siri/db/lookup.h>
+
+void siridb_pools_init(siridb_t * siridb);
+void siridb_pools_free(siridb_pools_t * pools);
+siridb_pool_t * siridb_pools_append(
+ siridb_pools_t * pools,
+ siridb_server_t * server);
+siridb_lookup_t * siridb_pools_gen_lookup(uint_fast16_t num_pools);
+int siridb_pools_online(siridb_t * siridb);
+int siridb_pools_available(siridb_t * siridb);
+int siridb_pools_accessible(siridb_t * siridb);
+void siridb_pools_send_pkg(
+ siridb_t * siridb,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promises_cb cb,
+ void * data,
+ int flags);
+void siridb_pools_send_pkg_2some(
+ vec_t * vec,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promises_cb cb,
+ void * data,
+ int flags);
+
+struct siridb_pools_s
+{
+ uint16_t len;
+ siridb_pool_t * pool;
+ siridb_lookup_t * lookup;
+ siridb_lookup_t * prev_lookup;
+};
+
+#endif /* SIRIDB_POOLS_H_ */
--- /dev/null
+/*
+ * presuf.h - Prefix and Suffix store.
+ */
+#ifndef SIRIDB_PRESUF_H_
+#define SIRIDB_PRESUF_H_
+
+typedef struct siridb_presuf_s siridb_presuf_t;
+
+#include <cleri/cleri.h>
+
+void siridb_presuf_free(siridb_presuf_t * presuf);
+siridb_presuf_t * siridb_presuf_add(
+ siridb_presuf_t ** presuf,
+ cleri_node_t * node);
+int siridb_presuf_is_unique(siridb_presuf_t * presuf);
+void siridb_presuf_cleanup(void);
+const char * siridb_presuf_name(
+ siridb_presuf_t * presuf,
+ const char * name,
+ size_t len);
+
+struct siridb_presuf_s
+{
+ char * prefix;
+ char * suffix;
+ size_t len; /* prefix len + suffix len + terminator char */
+ siridb_presuf_t * prev;
+};
+
+#endif /* SIRIDB_PRESUF_H_ */
--- /dev/null
+/*
+ * props.h - Functions to return SiriDB properties.
+ */
+#ifndef SIRIDB_PROPS_H_
+#define SIRIDB_PROPS_H_
+
+#include <siri/grammar/gramp.h>
+#include <siri/db/db.h>
+#include <qpack/qpack.h>
+
+void siridb_init_props(void);
+
+
+typedef void (* siridb_props_cb)(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int flags);
+
+siridb_props_cb props_get_cb(int i);
+void props_set_who_am_i(char * s);
+char * props_get_who_am_i(void);
+
+
+#endif /* SIRIDB_PROPS_H_ */
--- /dev/null
+/*
+ * queries.h - Query helpers for listener.
+ */
+#ifndef SIRIDB_QUERIES_H_
+#define SIRIDB_QUERIES_H_
+
+#include <uv.h>
+#include <inttypes.h>
+#include <imap/imap.h>
+#include <vec/vec.h>
+#include <cexpr/cexpr.h>
+#include <cleri/cleri.h>
+#include <ctree/ctree.h>
+#include <siri/db/group.h>
+#include <siri/db/presuf.h>
+#include <siri/db/series.h>
+#include <siri/db/tag.h>
+#include <siri/db/user.h>
+#include <pcre2.h>
+
+#define QUERIES_IGNORE_DROP_THRESHOLD 1
+#define QUERIES_SKIP_GET_POINTS 2
+
+enum
+{
+ QUERIES_ALTER,
+ QUERIES_COUNT,
+ QUERIES_DROP,
+ QUERIES_LIST,
+ QUERIES_SELECT
+};
+
+typedef enum
+{
+ QUERY_ALTER_NONE,
+ QUERY_ALTER_DATABASE,
+ QUERY_ALTER_GROUP,
+ QUERY_ALTER_TAG,
+ QUERY_ALTER_SERVER,
+ QUERY_ALTER_SERVERS,
+ QUERY_ALTER_USER,
+ QUERY_ALTER_SERIES
+} query_alter_tp;
+
+#define QUERY_DEF \
+uint8_t tp; \
+uint8_t flags; \
+imap_t * series_map; \
+imap_t * series_tmp; \
+imap_t * pmap; \
+vec_t * sset_vec; \
+vec_t * vec; \
+size_t vec_index; \
+imap_update_cb update_cb; \
+cexpr_t * where_expr; \
+pcre2_code * regex; \
+pcre2_match_data * match_data;
+
+typedef struct query_wrapper_s query_wrapper_t;
+typedef union query_alter_u query_alter_via_t;
+typedef struct query_alter_s query_alter_t;
+typedef struct query_count_s query_count_t;
+typedef struct query_drop_s query_drop_t;
+typedef struct query_list_s query_list_t;
+typedef struct query_select_s query_select_t;
+
+query_alter_t * query_alter_new(void);
+void query_alter_free(uv_handle_t * handle);
+
+query_count_t * query_count_new(void);
+void query_count_free(uv_handle_t * handle);
+
+query_drop_t * query_drop_new(void);
+void query_drop_free(uv_handle_t * handle);
+
+query_list_t * query_list_new(void);
+void query_list_free(uv_handle_t * handle);
+
+query_select_t * query_select_new(void);
+void query_select_free(uv_handle_t * handle);
+
+void query_help_free(uv_handle_t * handle);
+
+/* wrappers */
+struct query_wrapper_s
+{
+ QUERY_DEF
+};
+
+union query_alter_u
+{
+ siridb_group_t * group;
+ siridb_tag_t * tag;
+ siridb_server_t * server;
+ siridb_user_t * user;
+ void * dummy;
+};
+
+struct query_alter_s
+{
+ QUERY_DEF
+ query_alter_tp alter_tp;
+ query_alter_via_t via;
+ size_t n; /* can be used as counter */
+};
+
+struct query_count_s
+{
+ QUERY_DEF
+ size_t n; /* can be used as counter */
+};
+
+struct query_drop_s
+{
+ QUERY_DEF
+ size_t n; /* keep a counter for number of drops. */
+ vec_t * shards_list;
+};
+
+struct query_list_s
+{
+ QUERY_DEF
+ vec_t * props; /* will be freed */
+ size_t limit;
+};
+
+struct query_select_s
+{
+ QUERY_DEF
+ size_t n;
+ size_t nselects;
+ uint64_t * start_ts; /* will NOT be freed */
+ uint64_t * end_ts; /* will NOT be freed */
+ siridb_presuf_t * presuf;
+ char * merge_as;
+ ct_t * result;
+ imap_t * points_map; /* points_map for caching */
+ vec_t * alist; /* aggregation list (can be used multiple times)*/
+ vec_t * mlist; /* merge aggregation list */
+};
+
+#endif /* SIRIDB_QUERIES_H_ */
--- /dev/null
+/*
+ * query.h - Responsible for parsing queries.
+ */
+#ifndef SIRIDB_QUERY_H_
+#define SIRIDB_QUERY_H_
+
+#define SIRIDB_QUERY_FLAG_MASTER 1
+#define SIRIDB_QUERY_FLAG_REBUILD 2
+#define SIRIDB_QUERY_FLAG_UPDATE_REPLICA 4
+#define SIRIDB_QUERY_FLAG_ERR 8
+
+/*
+ * Note(*) : servers must be 'accessible' unless FLAG_ONLY_CHECK_ONLINE is used
+ */
+typedef enum
+{
+ SIRIDB_QUERY_FWD_SERVERS, /* Forward to all 'online' servers */
+ SIRIDB_QUERY_FWD_POOLS, /* Forward to all pools(*) */
+ SIRIDB_QUERY_FWD_SOME_POOLS, /* Forward to some pools(*) */
+ SIRIDB_QUERY_FWD_UPDATE /* Forward to all pools, upd repl(*) */
+} siridb_query_fwd_t;
+
+typedef enum siridb_err_tp
+{
+ SIRIDB_SUCCESS,
+
+ /* Server Errors 100 - 199 */
+ SIRIDB_ERR_SHUTTINGDOWN=100,
+
+ /* Query Errors 200 - 999 */
+ SIRIDB_ERR_INVALID_QUERY=200
+
+} siridb_err_t;
+
+typedef struct siridb_query_s siridb_query_t;
+
+#include <uv.h>
+#include <inttypes.h>
+#include <sys/time.h>
+#include <cleri/cleri.h>
+#include <qpack/qpack.h>
+#include <siri/db/time.h>
+#include <siri/db/nodes.h>
+#include <siri/db/series.h>
+#include <siri/db/db.h>
+#include <siri/net/protocol.h>
+#include <siri/inc.h>
+
+#if SIRIDB_EXPR_ALLOC
+#include <llist/llist.h>
+#endif
+
+void siridb_query_run(
+ uint16_t pid,
+ sirinet_stream_t * client,
+ const char * q,
+ size_t q_len,
+ float factor,
+ int flags);
+void siridb_query_free(uv_handle_t * handle);
+void siridb_send_query_result(uv_async_t * handle);
+void siridb_query_send_error(
+ uv_async_t * handle,
+ cproto_server_t err);
+void siridb_query_forward(
+ uv_async_t * handle,
+ siridb_query_fwd_t fwd,
+ sirinet_promises_cb cb,
+ int flags);
+void siridb_query_timeit_from_unpacker(
+ siridb_query_t * query,
+ qp_unpacker_t * unpacker);
+int siridb_query_err_from_pkg(siridb_query_t * query, sirinet_pkg_t * pkg);
+
+struct siridb_query_s
+{
+ uv_close_cb free_cb; /* must be on top */
+ uint8_t ref;
+ uint8_t flags;
+ uint16_t pid;
+ float factor;
+ void * data;
+ sirinet_stream_t * client;
+ char * q;
+ char err_msg[SIRIDB_MAX_SIZE_ERR_MSG];
+ qp_packer_t * packer;
+ qp_packer_t * timeit;
+ cleri_parse_t * pr;
+ siridb_nodes_t * nodes;
+ struct timespec start;
+#if SIRIDB_EXPR_ALLOC
+ llist_t * expr_cache;
+#endif
+};
+
+#endif /* SIRIDB_QUERY_H_ */
--- /dev/null
+/*
+ * re.h - Helpers for regular expressions.
+ */
+#ifndef SIRIDB_RE_H_
+#define SIRIDB_RE_H_
+
+#define PCRE2_CODE_UNIT_WIDTH 8
+
+#include <pcre2.h>
+#include <stddef.h>
+
+int siridb_re_compile(
+ pcre2_code ** regex,
+ pcre2_match_data ** match_data,
+ const char * source,
+ size_t len,
+ char * err_msg);
+
+
+#endif /* SIRIDB_RE_H_ */
--- /dev/null
+/*
+ * reindex.h - SiriDB Re-index.
+ *
+ * Differences while re-indexing:
+ *
+ * - Group information like number of series will be updated at a lower
+ * interval which leads to probably incorrect number of series per group.
+ * Selections for series in a group or a list of series per group are still
+ * correct and can only lack of brand new series. (newer than 30 seconds)
+ *
+ * - Selecting an unknown series usually raises a QueryError but we do not
+ * raise this error during re-indexing since the series might be in either
+ * the old- or new pool. (selecting series during re-indexing has therefore
+ * the same behavior as a regular expression selection)
+ *
+ * - Drop server is not allowed while re-indexing.
+ */
+#ifndef SIRIDB_REINDEX_H_
+#define SIRIDB_REINDEX_H_
+
+#define REINDEX_FN ".reindex"
+
+typedef struct siridb_reindex_s siridb_reindex_t;
+
+#include <inttypes.h>
+#include <uv.h>
+#include <siri/db/db.h>
+#include <siri/db/series.h>
+
+siridb_reindex_t * siridb_reindex_open(siridb_t * siridb, int create_new);
+void siridb_reindex_fopen(siridb_reindex_t * reindex, const char * opentype);
+void siridb_reindex_free(siridb_reindex_t ** reindex);
+void siridb_reindex_status_update(siridb_t * siridb);
+void siridb_reindex_close(siridb_reindex_t * reindex);
+void siridb_reindex_start(uv_timer_t * timer);
+const char * siridb_reindex_progress(siridb_t * siridb);
+
+struct siridb_reindex_s
+{
+ FILE * fp;
+ char * fn;
+ int fd;
+ long int size;
+ uint32_t * next_series_id;
+ sirinet_pkg_t * pkg_points;
+ sirinet_pkg_t * pkg_tags;
+ siridb_series_t * series;
+ siridb_server_t * server;
+ uv_timer_t * timer;
+};
+
+#endif /* SIRIDB_REINDEX_H_ */
--- /dev/null
+/*
+ * replicate.h - Replicate SiriDB.
+ */
+#ifndef SIRIDB_REPLICATE_H_
+#define SIRIDB_REPLICATE_H_
+
+typedef enum
+{
+ REPLICATE_IDLE,
+ REPLICATE_RUNNING,
+ REPLICATE_PAUSED,
+ REPLICATE_STOPPING,
+ REPLICATE_CLOSED
+} siridb_replicate_status_t;
+
+typedef struct siridb_replicate_s siridb_replicate_t;
+
+#include <uv.h>
+#include <siri/db/db.h>
+#include <siri/db/initsync.h>
+#include <siri/net/pkg.h>
+
+int siridb_replicate_init(siridb_t * siridb, siridb_initsync_t * initsync);
+void siridb_replicate_free(siridb_replicate_t ** replicate);
+int siridb_replicate_finish_init(siridb_replicate_t * replicate);
+void siridb_replicate_start(siridb_replicate_t * replicate);
+void siridb_replicate_close(siridb_replicate_t * replicate);
+void siridb_replicate_pause(siridb_replicate_t * replicate);
+void siridb_replicate_continue(siridb_replicate_t * replicate);
+int siridb_replicate_pkg(siridb_t * siridb, sirinet_pkg_t * pkg);
+sirinet_pkg_t * siridb_replicate_pkg_filter(
+ siridb_t * siridb,
+ unsigned char * data,
+ size_t len,
+ int flags);
+
+#define siridb_replicate_is_idle(replicate) (replicate->status == REPLICATE_IDLE)
+
+struct siridb_replicate_s
+{
+ siridb_replicate_status_t status;
+ uv_timer_t * timer;
+ siridb_initsync_t * initsync;
+};
+
+#endif /* SIRIDB_REPLICATE_H_ */
--- /dev/null
+/*
+ * series.c - SiriDB Time Series.
+ *
+ *
+ * Info siridb->series_mutex:
+ *
+ * Main thread:
+ * siridb->series_map : read (no lock) write (lock)
+ * series->idx : read (lock) write (lock)
+ *
+ * Other threads:
+ * siridb->series_map : read (lock) write (not allowed)
+ * series->idx : read (lock) write (lock)
+ *
+ * Note: One exception to 'not allowed' are the free functions
+ * since they only run when no other references to the object exist.
+ */
+#ifndef SIRIDB_SERIES_H_
+#define SIRIDB_SERIES_H_
+
+/* Series Flags */
+#define SIRIDB_SERIES_HAS_OVERLAP 1
+#define SIRIDB_SERIES_IS_DROPPED 2
+#define SIRIDB_SERIES_INIT_REPL 4
+#define SIRIDB_SERIES_IS_SERVER_ONE 8 /* if not set its server_id 0 */
+#define SIRIDB_SERIES_IS_32BIT_TS 16 /* if not set its a 64 bit ts */
+
+/* the max length including terminator char */
+#define SIRIDB_SERIES_NAME_LEN_MAX 65535
+
+#define siridb_series_isnum(series) (series->tp != TP_STRING)
+
+#define SIRIDB_QP_MAP2_TP(TP) \
+ (TP == QP_INT64) ? TP_INT : \
+ (TP == QP_DOUBLE) ? TP_DOUBLE : TP_STRING
+
+extern const char series_type_map[3][8];
+
+typedef struct idx_s idx_t;
+typedef struct siridb_series_s siridb_series_t;
+
+#include <inttypes.h>
+
+#include <siri/db/points.h>
+typedef points_tp series_tp;
+
+#include <siri/db/db.h>
+#include <siri/db/pcache.h>
+#include <siri/db/buffer.h>
+#include <qpack/qpack.h>
+#include <cexpr/cexpr.h>
+
+/* order here matters since shard.h is using a full series definition */
+struct siridb_series_s
+{
+ uint32_t ref; /* keep ref on top */
+ uint32_t id;
+ uint16_t mask;
+ uint16_t pool;
+ uint16_t name_len;
+ uint8_t flags;
+ uint8_t tp;
+ uint64_t start;
+ uint64_t end;
+ uint32_t length;
+ uint32_t idx_len;
+ long int bf_offset;
+ siridb_points_t * buffer;
+ char * name;
+ idx_t * idx;
+ siridb_t * siridb;
+};
+
+#include <siri/db/shard.h>
+
+int siridb_series_load(siridb_t * siridb);
+siridb_series_t * siridb_series_new(
+ siridb_t * siridb,
+ const char * series_name,
+ uint8_t tp);
+int siridb_series_add_idx(
+ siridb_series_t *__restrict series,
+ siridb_shard_t *__restrict shard,
+ uint64_t start_ts,
+ uint64_t end_ts,
+ uint32_t pos,
+ uint16_t len,
+ uint16_t cinfo);
+void series_update_start_end(siridb_series_t * series);
+int siridb_series_add_point(
+ siridb_t *__restrict siridb,
+ siridb_series_t *__restrict series,
+ uint64_t * ts,
+ qp_via_t * val);
+int siridb_series_add_pcache(
+ siridb_t *__restrict siridb,
+ siridb_series_t *__restrict series,
+ siridb_pcache_t *__restrict pcache);
+siridb_points_t * siridb_series_get_points(
+ siridb_series_t *__restrict series,
+ uint64_t *__restrict start_ts,
+ uint64_t *__restrict end_ts);
+void siridb_series_remove_shard(
+ siridb_t *__restrict siridb,
+ siridb_series_t *__restrict series,
+ siridb_shard_t *__restrict shard);
+int siridb_series_optimize_shard(
+ siridb_t *__restrict siridb,
+ siridb_series_t *__restrict series,
+ siridb_shard_t *__restrict shard);
+void siridb_series_update_props(siridb_t * siridb, siridb_series_t * series);
+int siridb_series_cexpr_cb(siridb_series_t * series, cexpr_condition_t * cond);
+int siridb_series_replicate_file(siridb_t * siridb);
+int siridb_series_drop(siridb_t * siridb, siridb_series_t * series);
+void siridb_series_drop_prepare(siridb_t * siridb, siridb_series_t * series);
+int siridb_series_drop_commit(siridb_t * siridb, siridb_series_t * series);
+int siridb_series_flush_dropped(siridb_t * siridb);
+uint8_t siridb_series_server_id_by_name(const char * name);
+int siridb_series_open_store(siridb_t * siridb);
+void siridb__series_free(siridb_series_t *__restrict series);
+void siridb__series_decref(siridb_series_t * series);
+siridb_points_t * siridb_series_get_first(
+ siridb_series_t * series, int * required_shard);
+siridb_points_t * siridb_series_get_last(
+ siridb_series_t * series, int * required_shard);
+siridb_points_t * siridb_series_get_count(siridb_series_t * series);
+void siridb_series_ensure_type(siridb_series_t * series, qp_obj_t * qp_obj);
+/*
+ * Increment the series reference counter.
+ */
+#define siridb_series_incref(series__) __atomic_add_fetch(&(series__)->ref, 1, __ATOMIC_SEQ_CST)
+/*
+ *
+ * Decrement reference counter for series and free the series when zero is
+ * reached.
+ */
+#define siridb_series_decref(series__) \
+ if (!__atomic_sub_fetch(&(series__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__series_free(series__)
+
+#define siridb_series_server_id(series) \
+((series->flags & SIRIDB_SERIES_IS_SERVER_ONE) == SIRIDB_SERIES_IS_SERVER_ONE)
+
+struct idx_s
+{
+ siridb_shard_t * shard;
+ uint32_t pos;
+ uint16_t len;
+ uint16_t cinfo; /* reserved for log values or used for compression */
+ uint64_t start_ts;
+ uint64_t end_ts;
+};
+
+#endif /* SIRIDB_SERIES_H_ */
--- /dev/null
+#include <siri/db/series.h>
+#include <siri/db/shard.h>
+
+static inline uint64_t siridb_series_duration(siridb_series_t * series)
+{
+ return series->idx_len ? series->idx->shard->duration : 0;
+}
--- /dev/null
+/*
+ * server.h - Each SiriDB database has at least one server.
+ */
+#ifndef SIRIDB_SERVER_H_
+#define SIRIDB_SERVER_H_
+
+#define FLAG_KEEP_PKG 1
+#define FLAG_ONLY_CHECK_ONLINE 2
+
+#define SERVER_FLAG_RUNNING 1
+#define SERVER_FLAG_SYNCHRONIZING 2
+#define SERVER_FLAG_REINDEXING 4
+#define SERVER_FLAG_BACKUP_MODE 8
+#define SERVER_FLAG_QUEUE_FULL 16 /* never set on 'this' server */
+#define SERVER_FLAG_UNAVAILABLE 32 /* never set on 'this' server */
+#define SERVER_FLAG_AUTHENTICATED 64 /* must be the last (we depend on this)
+ and will NEVER be set on 'this'
+ server */
+
+/* RUNNING + AUTHENTICATED */
+#define SERVER__IS_ONLINE 65
+
+/* RUNNING + SYNCHRONIZING + AUTHENTICATED */
+#define SERVER__IS_SYNCHRONIZING 67
+
+/* RUNNING + REINDEXING + AUTHENTICATED */
+#define SERVER__IS_REINDEXING 69
+
+#define SERVER__SELF_ONLINE 1 /* RUNNING */
+#define SERVER__SELF_SYNCHRONIZING 3 /* RUNNING + SYNCHRONIZING */
+#define SERVER__SELF_REINDEXING 5 /* RUNNING + REINDEXING */
+
+
+/*
+ * Server is 'connected' when at least connected.
+ */
+#define siridb_server_is_connected(server) \
+ (server->socket != NULL)
+
+/*
+ * Server is 'online' when at least running and authenticated but not
+ * queue-full. (unavailable status is intentionally ignored)
+ */
+#define siridb_server_is_online(server) \
+((server->flags & SERVER__IS_ONLINE) == SERVER__IS_ONLINE && \
+ (~server->flags & SERVER_FLAG_QUEUE_FULL))
+#define siridb_server_self_online(server) \
+((server->flags & SERVER__SELF_ONLINE) == SERVER__SELF_ONLINE)
+
+/*
+ * Server is 'available' when exactly running and authenticated.
+ */
+#define siridb_server_is_available(server) \
+(server->flags == SERVER__IS_ONLINE)
+#define siridb_server_self_available(server) \
+(server->flags == SERVER__SELF_ONLINE)
+
+/*
+ * Server is 'synchronizing' when exactly running, authenticated and
+ * synchronizing.
+ */
+#define siridb_server_is_synchronizing(server) \
+(server->flags == SERVER__IS_SYNCHRONIZING)
+#define siridb_server_self_synchronizing(server) \
+(server->flags == SERVER__SELF_SYNCHRONIZING)
+
+/*
+ * Server is 'accessible' when exactly running, authenticated and optionally
+ * re-indexing.
+ */
+#define siridb_server_is_accessible(server) \
+(server->flags == SERVER__IS_ONLINE || server->flags == SERVER__IS_REINDEXING)
+#define siridb_server_self_accessible(server) \
+(server->flags == SERVER__SELF_ONLINE || server->flags == SERVER__SELF_REINDEXING)
+
+typedef struct siridb_server_s siridb_server_t;
+typedef struct siridb_server_walker_s siridb_server_walker_t;
+typedef struct siridb_server_async_s siridb_server_async_t;
+
+#include <uuid/uuid.h>
+#include <stdint.h>
+#include <siri/db/db.h>
+#include <imap/imap.h>
+#include <cexpr/cexpr.h>
+#include <uv.h>
+#include <siri/net/promise.h>
+#include <siri/net/pkg.h>
+#include <siri/net/stream.h>
+
+siridb_server_t * siridb_server_new(
+ const char * uuid,
+ const char * address,
+ size_t address_len,
+ uint16_t port,
+ uint16_t pool);
+
+void siridb_server_connect(siridb_t * siridb, siridb_server_t * server);
+int siridb_server_send_pkg(
+ siridb_server_t * server,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promise_cb cb,
+ void * data,
+ int flags);
+void siridb_server_send_flags(siridb_server_t * server);
+int siridb_server_update_address(
+ siridb_t * siridb,
+ siridb_server_t * server,
+ const char * address,
+ uint16_t port);
+char * siridb_server_str_status(siridb_server_t * server);
+siridb_server_t * siridb_server_from_node(
+ siridb_t * siridb,
+ cleri_node_t * server_node,
+ char * err_msg);
+int siridb_server_drop(siridb_t * siridb, siridb_server_t * server);
+int siridb_server_is_remote_prop(uint32_t prop);
+int siridb_server_cexpr_cb(
+ siridb_server_walker_t * wserver,
+ cexpr_condition_t * cond);
+siridb_server_t * siridb_server_register(
+ siridb_t * siridb,
+ unsigned char * data,
+ size_t len);
+void siridb__server_free(siridb_server_t * server);
+
+/* This will remove the unavailable status but the authenticated and queue_full
+ * flags are kept.
+ */
+#define siridb_server_update_flags(org, new) \
+ org = new | (org & (SERVER_FLAG_AUTHENTICATED | SERVER_FLAG_QUEUE_FULL))
+
+#define siridb_server_incref(server__) __atomic_add_fetch(&(server__)->ref, 1, __ATOMIC_SEQ_CST)
+
+/*
+ * Decrement server reference counter and free the server when zero is reached.
+ * When the server is destroyed, all remaining server->promises are cancelled
+ * and each promise->cb() will be called.
+ */
+#define siridb_server_decref(server__) \
+ if (!__atomic_sub_fetch(&(server__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__server_free(server__)
+
+struct siridb_server_s
+{
+ uint16_t ref; /* keep ref on top */
+ uint16_t port;
+ uint16_t pool;
+ uint8_t flags; /* do not use flags above 16384 */
+ uint8_t id; /* set when added to a pool to either 0 or 1 */
+ char * name; /* this is a format for address:port but we use it a lot */
+ char * address;
+ imap_t * promises;
+ sirinet_stream_t * client;
+ uint16_t pid;
+ /* fixed server properties */
+ uint8_t ip_support;
+ uint8_t retry_attempts;
+ uint32_t startup_time;
+ char * libuv;
+ char * version;
+ char * dbpath;
+ char * buffer_path;
+ size_t buffer_size;
+ uuid_t uuid;
+};
+
+struct siridb_server_walker_s
+{
+ siridb_server_t * server;
+ siridb_t * siridb;
+};
+
+struct siridb_server_async_s
+{
+ uint16_t pid;
+ sirinet_stream_t * client;
+};
+
+/*
+ * Returns < 0 if the uuid from server A is less than uuid from server B.
+ * Returns > 0 if the uuid from server A is greater than uuid from server B.
+ * Returns 0 when uuid server A and B are equal.
+ */
+static inline int siridb_server_cmp(siridb_server_t * sa, siridb_server_t * sb)
+{
+ return uuid_compare(sa->uuid, sb->uuid);
+}
+
+#endif /* SIRIDB_SERVER_H_ */
--- /dev/null
+/*
+ * servers.h - Collection of SiriDB servers.
+ */
+#ifndef SIRIDB_SERVERS_H_
+#define SIRIDB_SERVERS_H_
+
+#include <siri/db/db.h>
+#include <uuid/uuid.h> /* install: apt-get install uuid-dev */
+#include <siri/net/promise.h>
+
+int siridb_servers_load(siridb_t * siridb);
+void siridb_servers_free(llist_t * servers);
+siridb_server_t * siridb_servers_by_uuid(llist_t * servers, uuid_t uuid);
+siridb_server_t * siridb_servers_by_name(llist_t * servers, const char * name);
+siridb_server_t * siridb_servers_by_replica(
+ llist_t * servers,
+ siridb_server_t * replica);
+
+void siridb_servers_send_pkg(
+ vec_t * servers,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promises_cb cb,
+ void * data);
+void siridb_servers_send_flags(llist_t * servers);
+ssize_t siridb_servers_get_file(char ** buffer, siridb_t * siridb);
+int siridb_servers_online(siridb_t * siridb);
+int siridb_servers_available(siridb_t * siridb);
+int siridb_servers_list(siridb_server_t * server, uv_async_t * handle);
+int siridb_servers_check_version(siridb_t * siridb, char * version);
+int siridb_servers_save(siridb_t * siridb);
+int siridb_servers_register(siridb_t * siridb, siridb_server_t * server);
+vec_t * siridb_servers_other2vec(siridb_t * siridb);
+
+#endif /* SIRIDB_SERVERS_H_ */
--- /dev/null
+/*
+ * shard.h - SiriDB shard file.
+ */
+#ifndef SIRIDB_SHARD_H_
+#define SIRIDB_SHARD_H_
+
+
+
+/* flags */
+#define SIRIDB_SHARD_OK 0
+#define SIRIDB_SHARD_HAS_INDEX 1
+#define SIRIDB_SHARD_HAS_OVERLAP 2
+#define SIRIDB_SHARD_HAS_NEW_VALUES 4
+#define SIRIDB_SHARD_HAS_DROPPED_SERIES 8
+#define SIRIDB_SHARD_IS_REMOVED 16
+#define SIRIDB_SHARD_IS_LOADING 32
+#define SIRIDB_SHARD_IS_CORRUPT 64
+#define SIRIDB_SHARD_IS_COMPRESSED 128
+
+/* HAS_OVERLAP + HAS_NEW_VALUES + HAS_DROPPED_SERIES + IS_CORRUPT */
+#define SIRIDB_SHARD_NEED_OPTIMIZE 78
+
+/* types */
+#define SIRIDB_SHARD_TP_NUMBER 0
+#define SIRIDB_SHARD_TP_LOG 1
+
+#define SIRIDB_SHARD_STATUS_STR_MAX 128
+
+extern const char shard_type_map[2][7];
+
+typedef struct siridb_shard_flags_repr_s siridb_shard_flags_repr_t;
+typedef struct siridb_shard_s siridb_shard_t;
+typedef struct siridb_shard_view_s siridb_shard_view_t;
+
+#include <stdio.h>
+#include <siri/db/db.h>
+#include <siri/db/points.h>
+#include <siri/db/series.h>
+#include <siri/file/handler.h>
+#include <omap/omap.h>
+
+siridb_shard_t * siridb_shard_create(
+ siridb_t * siridb,
+ omap_t * shards,
+ uint64_t id,
+ uint64_t duration,
+ uint8_t tp,
+ siridb_shard_t * replacing);
+uint64_t siridb_shard_duration_from_interval(siridb_t * siridb, uint64_t interval);
+uint64_t siridb_shard_interval_from_duration(uint64_t duration);
+int siridb_shard_cexpr_cb(
+ siridb_shard_view_t * vshard,
+ cexpr_condition_t * cond);
+int siridb_shard_status(char * str, siridb_shard_t * shard);
+int siridb_shard_load(siridb_t * siridb, uint64_t id, uint64_t duration);
+void siridb_shard_drop(siridb_shard_t * shard, siridb_t * siridb);
+size_t siridb_shard_write_points(
+ siridb_t * siridb,
+ siridb_series_t * series,
+ siridb_shard_t * shard,
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ FILE * idx_fp,
+ uint16_t * cinfo);
+typedef int (*siridb_shard_get_points_cb)(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_shard_get_points_num32(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_shard_get_points_num64(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_shard_get_points_log32(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_shard_get_points_log64(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_shard_get_points_num_compressed(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_shard_get_points_log_compressed(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+int siridb_shard_migrate(
+ siridb_t * siridb,
+ uint64_t shard_id,
+ uint64_t * duration);
+int siridb_shard_optimize(siridb_shard_t * shard, siridb_t * siridb);
+void siridb__shard_free(siridb_shard_t * shard);
+void siridb__shard_decref(siridb_shard_t * shard);
+
+struct siridb_shard_flags_repr_s
+{
+ const char * repr;
+ uint8_t flag;
+};
+
+struct siridb_shard_s
+{
+ uint32_t ref; /* keep ref on top */
+ uint8_t tp; /* TP_NUMBER, TP_LOG */
+ uint8_t flags;
+ uint16_t max_chunk_sz;
+ uint64_t id;
+ size_t len; /* size of the shard which is used */
+ size_t size; /* size of shard on disk */
+ uint64_t duration; /* based on the interval of series */
+ siri_fp_t * fp;
+ char * fn;
+ siridb_shard_t * replacing;
+};
+
+struct siridb_shard_view_s
+{
+ siridb_shard_t * shard;
+ siridb_server_t * server;
+ uint64_t start;
+ uint64_t end;
+};
+
+static inline siridb_shard_get_points_cb siridb_shard_get_points_callback(
+ uint8_t shard_flags,
+ siridb_series_t * series)
+{
+ return shard_flags & SIRIDB_SHARD_IS_COMPRESSED ?
+ (series->tp == TP_STRING ?
+ siridb_shard_get_points_log_compressed :
+ siridb_shard_get_points_num_compressed) :
+ (series->tp == TP_STRING ?
+ (series->flags & SIRIDB_SERIES_IS_32BIT_TS ?
+ siridb_shard_get_points_log32 :
+ siridb_shard_get_points_log64) :
+ (series->flags & SIRIDB_SERIES_IS_32BIT_TS ?
+ siridb_shard_get_points_num32 :
+ siridb_shard_get_points_num64));
+}
+
+/*
+ * Increment the shard reference counter.
+ */
+#define siridb_shard_incref(shard__) __atomic_add_fetch(&(shard__)->ref, 1, __ATOMIC_SEQ_CST)
+
+/*
+ * Decrement the reference counter, when 0 the shard will be destroyed.
+ *
+ * In case the shard will be destroyed and flag SIRIDB_SHARD_WILL_BE_REMOVED
+ * is set, the file will be removed.
+ *
+ * A signal can be raised in case closing the shard file fails.
+ */
+#define siridb_shard_decref(shard__) \
+ if (!__atomic_sub_fetch(&(shard__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__shard_free(shard__)
+
+
+#define siridb_shard_idx_file(Name__, Fn__) \
+ size_t Len__ = strlen(Fn__); \
+ char Name__[Len__ + 1]; \
+ memcpy(Name__, Fn__, Len__ - 3); \
+ memcpy(Name__ + Len__ - 3, "idx", 4)
+
+#endif /* SIRIDB_SHARD_H_ */
--- /dev/null
+/*
+ * shards.h - Collection of SiriDB shards.
+ *
+ * Info shards->mutex:
+ *
+ * Main thread:
+ * siridb->shards : read (lock) write (lock)
+
+ * Other threads:
+ * siridb->shards : read (lock) write (lock)
+ *
+ * Note: since series->idx hold a reference to a shard, a lock to the
+ * series_mutex is required in some cases.
+ */
+#ifndef SIRIDB_SHARDS_H_
+#define SIRIDB_SHARDS_H_
+
+#define SIRIDB_SHARDS_PATH "shards/"
+
+#include <siri/db/db.h>
+#include <omap/omap.h>
+
+void siridb_shards_destroy_cb(omap_t * shards);
+int siridb_shards_load(siridb_t * siridb);
+int siridb_shards_add_points(
+ siridb_t * siridb,
+ siridb_series_t * series,
+ siridb_points_t * points);
+double siridb_shards_count_percent(
+ siridb_t * siridb,
+ uint64_t end_ts,
+ uint8_t tp);
+size_t siridb_shards_n(siridb_t * siridb);
+vec_t * siridb_shards_vec(siridb_t * siridb);
+
+#endif /* SIRIDB_SHARDS_H_ */
--- /dev/null
+/*
+ * sset.h - Set operations on series.
+ */
+#ifndef SIRIDB_SSET_H_
+#define SIRIDB_SSET_H_
+
+typedef struct siridb_sset_s siridb_sset_t;
+
+#include <imap/imap.h>
+
+siridb_sset_t * siridb_sset_new(imap_t * series_map, imap_update_cb update_cb);
+void siridb_sset_free(siridb_sset_t * sset);
+
+struct siridb_sset_s
+{
+ imap_t * series_map;
+ imap_update_cb update_cb;
+};
+
+#endif /* SIRIDB_SSET_H_ */
--- /dev/null
+/*
+ * tag.h - Tag (tag series).
+ */
+#ifndef SIRIDB_TAG_H_
+#define SIRIDB_TAG_H_
+
+typedef struct siridb_tag_s siridb_tag_t;
+
+enum
+{
+ TAG_FLAG_CLEANUP = 1<<0,
+ TAG_FLAG_REQUIRE_SAVE = 1<<1,
+};
+
+#include <inttypes.h>
+#include <imap/imap.h>
+#include <siri/db/db.h>
+
+siridb_tag_t * siridb_tag_new(siridb_tags_t * tags, uint64_t id);
+void siridb__tag_decref(siridb_tag_t * tag);
+void siridb__tag_free(siridb_tag_t * tag);
+int siridb_tag_is_valid_fn(const char * fn);
+siridb_tag_t * siridb_tag_load(siridb_t * siridb, const char * _fn);
+int siridb_tag_save(siridb_tag_t * tag);
+char * siridb_tag_fn(siridb_tag_t * tag);
+int siridb_tag_is_remote_prop(uint32_t prop);
+void siridb_tag_prop(siridb_tag_t * tag, qp_packer_t * packer, int prop);
+int siridb_tag_cexpr_cb(siridb_tag_t * tag, cexpr_condition_t * cond);
+int siridb_tag_check_name(const char * name, char * err_msg);
+int siridb_tag_set_name(
+ siridb_t * siridb,
+ siridb_tag_t * tag,
+ const char * name,
+ char * err_msg);
+
+#define siridb_tag_incref(tag__) __atomic_add_fetch(&(tag__)->ref, 1, __ATOMIC_SEQ_CST)
+#define siridb_tag_decref(tag__) \
+ if (!__atomic_sub_fetch(&(tag__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__tag_free(tag__)
+
+struct siridb_tag_s
+{
+ uint16_t ref;
+ uint16_t flags;
+ uint32_t n;
+ uint64_t id;
+ char * name;
+ siridb_tags_t * tags;
+ imap_t * series;
+};
+
+
+#endif /* SIRIDB_TAG_H_ */
--- /dev/null
+/*
+ * tags.h - Tag (tagged series).
+ */
+#ifndef SIRIDB_TAGS_H_
+#define SIRIDB_TAGS_H_
+
+typedef struct siridb_tags_s siridb_tags_t;
+
+#include <inttypes.h>
+#include <ctree/ctree.h>
+#include <vec/vec.h>
+#include <uv.h>
+#include <siri/db/db.h>
+#include <siri/db/tag.h>
+
+#define SIRIDB_TAGS_PATH "tags/"
+
+enum
+{
+ TAGS_FLAG_DROPPED_SERIES = 1<<0,
+ TAGS_FLAG_REQUIRE_SAVE = 1<<1,
+};
+
+struct siridb_tags_s
+{
+ uint16_t flags;
+ uint16_t ref;
+ uint32_t pad0;
+ uint64_t next_id;
+ char * path;
+ ct_t * tags;
+ uv_mutex_t mutex;
+};
+
+int siridb_tags_init(siridb_t * siridb);
+int siridb_tags_drop_tag(
+ siridb_tags_t * tags,
+ const char * name,
+ char * err_msg);
+siridb_tag_t * siridb_tags_add_n(
+ siridb_tags_t * tags,
+ const char * name,
+ size_t name_len);
+siridb_tag_t * siridb_tags_add(siridb_tags_t * tags, const char * name);
+void siridb_tags_dropped_series(siridb_tags_t * tags);
+void siridb_tags_save(siridb_tags_t * tags);
+void siridb_tags_init_nseries(siridb_tags_t * tags);
+sirinet_pkg_t * siridb_tags_pkg(siridb_tags_t * tags, uint16_t pid);
+sirinet_pkg_t * siridb_tags_series(siridb_series_t * series);
+sirinet_pkg_t * siridb_tags_empty(siridb_tags_t * tags);
+void siridb__tags_free(siridb_tags_t * tags);
+
+#define siridb_tags_set_require_save(__tags, __tag) \
+do{ \
+ (__tags)->flags |= TAGS_FLAG_REQUIRE_SAVE; \
+ (__tag)->flags |= TAG_FLAG_REQUIRE_SAVE; \
+}while(0)
+
+
+#define siridb_tags_incref(tags__) __atomic_add_fetch(&(tags__)->ref, 1, __ATOMIC_SEQ_CST)
+#define siridb_tags_decref(tags__) \
+ if (!__atomic_sub_fetch(&(tags__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__tags_free(tags__)
+
+#endif /* SIRIDB_TAGS_H_ */
--- /dev/null
+/*
+ * tasks.f - Counters for info on SiriDB tasks.
+ */
+#ifndef SIRIDB_TASKS_H_
+#define SIRIDB_TASKS_H_
+
+typedef struct siridb_tasks_s siridb_tasks_t;
+
+#include <time.h>
+#include <inttypes.h>
+#include <timeit/timeit.h>
+
+void siridb_tasks_init(siridb_tasks_t * tasks);
+
+#define siridb_tasks_inc(tasks) \
+if (!tasks.active++) tasks.idle_time += timeit_get(&tasks._timeit)
+
+#define siridb_tasks_dec(tasks) \
+if (!--tasks.active) timeit_start(&tasks._timeit)
+
+struct siridb_tasks_s
+{
+ struct timespec _timeit;
+ uint64_t active;
+ double idle_time;
+};
+
+#endif /* SIRIDB_TASKS_H_ */
--- /dev/null
+/*
+ * tee.h - To tee the data for a SiriDB database.
+ */
+#ifndef SIRIDB_TEE_H_
+#define SIRIDB_TEE_H_
+
+typedef struct siridb_tee_s siridb_tee_t;
+
+enum
+{
+ SIRIDB_TEE_FLAG_INIT = 1<<0,
+ SIRIDB_TEE_FLAG_CONNECTING = 1<<1,
+ SIRIDB_TEE_FLAG_CONNECTED = 1<<2,
+ SIRIDB_TEE_FLAG = 1<<31,
+};
+
+#include <uv.h>
+#include <stdbool.h>
+#include <siri/net/promise.h>
+
+siridb_tee_t * siridb_tee_new(void);
+void siridb_tee_free(siridb_tee_t * tee);
+int siridb_tee_connect(siridb_tee_t * tee);
+int siridb_tee_set_pipe_name(siridb_tee_t * tee, const char * pipe_name);
+void siridb_tee_write(siridb_tee_t * tee, sirinet_promise_t * promise);
+const char * tee_str(siridb_tee_t * tee);
+static inline _Bool siridb_tee_is_configured(siridb_tee_t * tee);
+static inline _Bool siridb_tee_is_connected(siridb_tee_t * tee);
+static inline _Bool siridb_tee_is_handle(uv_handle_t * handle);
+
+struct siridb_tee_s
+{
+ uint32_t flags; /* maps to sirnet_stream_t tp for cleanup */
+ char * pipe_name_;
+ char * err_msg_;
+ uv_pipe_t pipe;
+};
+
+static inline _Bool siridb_tee_is_configured(siridb_tee_t * tee)
+{
+ return tee->pipe_name_ != NULL;
+};
+
+static inline _Bool siridb_tee_is_connected(siridb_tee_t * tee)
+{
+ return tee->flags & SIRIDB_TEE_FLAG_CONNECTED;
+}
+
+static inline _Bool siridb_tee_is_handle(uv_handle_t * handle)
+{
+ return
+ handle->type == UV_NAMED_PIPE &&
+ handle->data &&
+ (((siridb_tee_t *) handle->data)->flags & SIRIDB_TEE_FLAG);
+}
+
+#endif /* SIRIDB_TEE_H_ */
--- /dev/null
+/*
+ * time.h - Time- and time precision functions and constants.
+ */
+#ifndef SIRIDB_TIME_H_
+#define SIRIDB_TIME_H_
+
+typedef enum
+{
+ SIRIDB_TIME_DEFAULT=-1, /* use this when asking for the db default */
+ SIRIDB_TIME_SECONDS,
+ SIRIDB_TIME_MILLISECONDS,
+ SIRIDB_TIME_MICROSECONDS,
+ SIRIDB_TIME_NANOSECONDS,
+ SIRIDB_TIME_END
+} siridb_timep_t;
+
+typedef struct siridb_time_s siridb_time_t;
+
+#include <inttypes.h>
+#include <siri/db/db.h>
+#include <stddef.h>
+#include <time.h>
+
+static const char * SIRIDB_TIME_SHORT_MAP[SIRIDB_TIME_END] = {"s", "ms", "us", "ns"};
+siridb_time_t * siridb_time_new(siridb_timep_t precision);
+uint32_t siridb_time_in_seconds(siridb_t * siridb, int64_t ts);
+uint64_t siridb_time_now(siridb_t * siridb, struct timespec now);
+uint64_t siridb_time_parse(const char * str, size_t len);
+
+struct siridb_time_s
+{
+ siridb_timep_t precision;
+ uint32_t factor;
+ size_t ts_sz;
+};
+
+static inline int siridb_int64_valid_ts(siridb_time_t * time, int64_t ts)
+{
+ return (time->precision == SIRIDB_TIME_SECONDS) ?
+ ts >= 0 && ts < 4294967296 : ts >= 0;
+}
+
+static inline const char * siridb_time_short_map(siridb_timep_t tp)
+{
+ return SIRIDB_TIME_SHORT_MAP[tp];
+}
+
+#endif /* SIRIDB_TIME_H_ */
--- /dev/null
+/*
+ * user.h - Contains functions for a SiriDB database user.
+ */
+#ifndef SIRIDB_USER_H_
+#define SIRIDB_USER_H_
+
+typedef struct siridb_user_s siridb_user_t;
+
+#include <qpack/qpack.h>
+#include <inttypes.h>
+#include <siri/db/db.h>
+#include <siri/db/access.h>
+#include <cexpr/cexpr.h>
+
+siridb_user_t * siridb_user_new(void);
+void siridb_user_prop(siridb_user_t * user, qp_packer_t * packer, int prop);
+int siridb_user_set_name(
+ siridb_t * siridb,
+ siridb_user_t * user,
+ const char * name,
+ char * err_msg);
+int siridb_user_set_password(
+ siridb_user_t * user,
+ const char * password,
+ char * err_msg);
+
+int siridb_user_check_access(
+ siridb_user_t * user,
+ uint32_t access_bit,
+ char * err_msg);
+
+void siridb__user_free(siridb_user_t * user);
+
+int siridb_user_cexpr_cb(siridb_user_t * user, cexpr_condition_t * cond);
+
+/*
+ * Increment the user reference counter.
+ */
+#define siridb_user_incref(user__) __atomic_add_fetch(&(user__)->ref, 1, __ATOMIC_SEQ_CST)
+
+/*
+ * Decrement user reference counter and free the user when zero is reached.
+ */
+#define siridb_user_decref(user__) \
+ if (!__atomic_sub_fetch(&(user__)->ref, 1, __ATOMIC_SEQ_CST)) siridb__user_free(user__)
+
+struct siridb_user_s
+{
+ uint16_t ref;
+ uint16_t pad0;
+ uint32_t access_bit;
+ char * name;
+ char * password; /* keeps an encrypted password */
+};
+#endif /* SIRIDB_USER_H_ */
--- /dev/null
+/*
+ * users.h - Collection of database users.
+ */
+#ifndef SIRIDB_USERS_H_
+#define SIRIDB_USERS_H_
+
+#include <inttypes.h>
+#include <siri/db/db.h>
+#include <siri/db/user.h>
+#include <llist/llist.h>
+
+int siridb_users_load(siridb_t * siridb);
+void siridb_users_free(llist_t * users);
+int siridb_users_add_user(
+ siridb_t * siridb,
+ siridb_user_t * user,
+ char * err_msg);
+int siridb_users_drop_user(
+ siridb_t * siridb,
+ const char * username,
+ char * err_msg);
+siridb_user_t * siridb_users_get_user(
+ siridb_t * siridb,
+ const char * username,
+ const char * password);
+siridb_user_t * siridb_users_get_user_from_basic(
+ siridb_t * siridb,
+ const char * data,
+ size_t n);
+int siridb_users_save(siridb_t * siridb);
+ssize_t siridb_users_get_file(char ** buffer, siridb_t * siridb);
+
+#endif /* SIRIDB_USERS_H_ */
--- /dev/null
+/*
+ * variance.h - Calculate variance for points.
+ */
+#ifndef SIRIDB_VARIANCE_H_
+#define SIRIDB_VARIANCE_H_
+
+#include <siri/db/points.h>
+
+double siridb_variance(siridb_points_t * points);
+
+#endif /* SIRIDB_VARIANCE_H_ */
--- /dev/null
+/*
+ * walker.h - Creates enter and exit nodes.
+ */
+#ifndef SIRIDB_WALKER_H_
+#define SIRIDB_WALKER_H_
+
+typedef struct siridb_walker_s siridb_walker_t;
+
+#include <cleri/cleri.h>
+#include <siri/db/nodes.h>
+#include <uv.h>
+
+siridb_walker_t * siridb_walker_new(
+ siridb_t * siridb,
+ const uint64_t now,
+ uint8_t * flags);
+
+/* free the walker and return the nodes which are kept in the walker.
+ * note: the nodes are created using 'malloc()' so they must be destroyed
+ * using siridb_nodes_free().
+ */
+siridb_nodes_t * siridb_walker_free(siridb_walker_t * walker);
+
+int siridb_walker_append(
+ siridb_walker_t * walker,
+ cleri_node_t * node,
+ uv_async_cb cb);
+int siridb_walker_insert(
+ siridb_walker_t * walker,
+ cleri_node_t * node,
+ uv_async_cb cb);
+
+struct siridb_walker_s
+{
+ siridb_t * siridb;
+ uint64_t now;
+ uint8_t * flags;
+ siridb_nodes_t * start;
+ siridb_nodes_t * enter_nodes;
+ siridb_nodes_t * exit_nodes;
+};
+
+#endif /* SIRIDB_WALKER_H_ */
--- /dev/null
+/*
+ * err.h - SiriDB Error.
+ */
+#ifndef SIRI_ERR_H_
+#define SIRI_ERR_H_
+
+#include <logger/logger.h>
+#include <signal.h>
+
+
+/* value should be 0,
+ * any other value indicates a critical error has occurred */
+extern int siri_err;
+
+#define ERR_CLOSE_ENFORCED -3
+#define ERR_CLOSE_TIMEOUT_REACHED -2
+#define ERR_STARTUP -1
+
+#define ERR_ALLOC \
+log_critical("Memory allocation error at: %s:%d (%s)", \
+ __FILE__, __LINE__, __func__); \
+raise(SIGSEGV); \
+if (!siri_err) siri_err = SIGSEGV;
+
+#define ERR_FILE \
+log_critical("Critical file error at: %s:%d (%s)", \
+ __FILE__, __LINE__, __func__); \
+raise(SIGABRT); \
+if (!siri_err) siri_err = SIGABRT;
+
+#define ERR_C \
+log_critical("Critical error at: %s:%d (%s)", \
+ __FILE__, __LINE__, __func__); \
+raise(SIGABRT); \
+if (!siri_err) siri_err = SIGABRT;
+
+#endif /* SIRI_ERR_H_ */
--- /dev/null
+/*
+ * siri/evars.h
+ */
+#ifndef SIRI_EVARS_H_
+#define SIRI_EVARS_H_
+
+#include <siri/siri.h>
+
+void siri_evars_parse(siri_t * siri);
+
+#endif /* SIRI_EVARS_H_ */
--- /dev/null
+/*
+ * handler.h - File handler for shard files.
+ */
+#ifndef SIRI_FH_H_
+#define SIRI_FH_H_
+
+typedef struct siri_fh_s siri_fh_t;
+
+#include <inttypes.h>
+#include <siri/file/pointer.h>
+
+siri_fh_t * siri_fh_new(uint16_t size);
+void siri_fh_free(siri_fh_t * fh);
+int siri_fopen(
+ siri_fh_t * fh,
+ siri_fp_t * fp,
+ const char * fn,
+ const char * modes);
+
+struct siri_fh_s
+{
+ uint16_t size;
+ uint16_t idx;
+ siri_fp_t ** fpointers;
+};
+
+#endif /* SIRI_FH_H_ */
--- /dev/null
+/*
+ * pointer.h - File pointer used in combination with file handler.
+ */
+#ifndef SIRI_FP_H_
+#define SIRI_FP_H_
+
+typedef struct siri_fp_s siri_fp_t;
+
+#include <stdio.h>
+#include <inttypes.h>
+
+siri_fp_t * siri_fp_new(void);
+/* closes the file pointer, decrement reference counter and free if needed */
+void siri_fp_decref(siri_fp_t * fp);
+void siri_fp_close(siri_fp_t * fp);
+
+struct siri_fp_s
+{
+ FILE * fp;
+ uint8_t ref;
+};
+
+#endif /* SIRI_FP_H_ */
--- /dev/null
+/*
+ * siri/grammar/grammar.h
+ *
+ * This grammar is generated using the Grammar.export_c() method and
+ * should be used with the libcleri module.
+ *
+ * Source class: SiriGrammar
+ * Created at: 2020-09-25 10:57:26
+ */
+#ifndef CLERI_EXPORT_SIRI_GRAMMAR_GRAMMAR_H_
+#define CLERI_EXPORT_SIRI_GRAMMAR_GRAMMAR_H_
+
+#include <cleri/cleri.h>
+
+cleri_grammar_t * compile_siri_grammar_grammar(void);
+
+enum cleri_grammar_ids {
+ CLERI_NONE, // used for objects with no name
+ CLERI_GID_ACCESS_EXPR,
+ CLERI_GID_ACCESS_KEYWORDS,
+ CLERI_GID_AFTER_EXPR,
+ CLERI_GID_AGGREGATE_FUNCTIONS,
+ CLERI_GID_ALTER_DATABASE,
+ CLERI_GID_ALTER_GROUP,
+ CLERI_GID_ALTER_SERIES,
+ CLERI_GID_ALTER_SERVER,
+ CLERI_GID_ALTER_SERVERS,
+ CLERI_GID_ALTER_STMT,
+ CLERI_GID_ALTER_TAG,
+ CLERI_GID_ALTER_USER,
+ CLERI_GID_BEFORE_EXPR,
+ CLERI_GID_BETWEEN_EXPR,
+ CLERI_GID_BOOL_OPERATOR,
+ CLERI_GID_CALC_STMT,
+ CLERI_GID_COUNT_GROUPS,
+ CLERI_GID_COUNT_POOLS,
+ CLERI_GID_COUNT_SERIES,
+ CLERI_GID_COUNT_SERIES_LENGTH,
+ CLERI_GID_COUNT_SERVERS,
+ CLERI_GID_COUNT_SERVERS_RECEIVED,
+ CLERI_GID_COUNT_SERVERS_SELECTED,
+ CLERI_GID_COUNT_SHARDS,
+ CLERI_GID_COUNT_SHARDS_SIZE,
+ CLERI_GID_COUNT_STMT,
+ CLERI_GID_COUNT_TAGS,
+ CLERI_GID_COUNT_USERS,
+ CLERI_GID_CREATE_GROUP,
+ CLERI_GID_CREATE_STMT,
+ CLERI_GID_CREATE_USER,
+ CLERI_GID_C_DIFFERENCE,
+ CLERI_GID_DROP_GROUP,
+ CLERI_GID_DROP_SERIES,
+ CLERI_GID_DROP_SERVER,
+ CLERI_GID_DROP_SHARDS,
+ CLERI_GID_DROP_STMT,
+ CLERI_GID_DROP_TAG,
+ CLERI_GID_DROP_USER,
+ CLERI_GID_F_ALL,
+ CLERI_GID_F_COUNT,
+ CLERI_GID_F_DERIVATIVE,
+ CLERI_GID_F_DIFFERENCE,
+ CLERI_GID_F_FILTER,
+ CLERI_GID_F_FIRST,
+ CLERI_GID_F_INTERVAL,
+ CLERI_GID_F_LAST,
+ CLERI_GID_F_LIMIT,
+ CLERI_GID_F_MAX,
+ CLERI_GID_F_MEAN,
+ CLERI_GID_F_MEDIAN,
+ CLERI_GID_F_MEDIAN_HIGH,
+ CLERI_GID_F_MEDIAN_LOW,
+ CLERI_GID_F_MIN,
+ CLERI_GID_F_POINTS,
+ CLERI_GID_F_PVARIANCE,
+ CLERI_GID_F_STDDEV,
+ CLERI_GID_F_SUM,
+ CLERI_GID_F_TIMEVAL,
+ CLERI_GID_F_VARIANCE,
+ CLERI_GID_GRANT_STMT,
+ CLERI_GID_GRANT_USER,
+ CLERI_GID_GROUP_COLUMNS,
+ CLERI_GID_GROUP_NAME,
+ CLERI_GID_GROUP_TAG_MATCH,
+ CLERI_GID_HELP_ACCESS,
+ CLERI_GID_HELP_ALTER,
+ CLERI_GID_HELP_ALTER_DATABASE,
+ CLERI_GID_HELP_ALTER_GROUP,
+ CLERI_GID_HELP_ALTER_SERVER,
+ CLERI_GID_HELP_ALTER_SERVERS,
+ CLERI_GID_HELP_ALTER_USER,
+ CLERI_GID_HELP_COUNT,
+ CLERI_GID_HELP_COUNT_GROUPS,
+ CLERI_GID_HELP_COUNT_POOLS,
+ CLERI_GID_HELP_COUNT_SERIES,
+ CLERI_GID_HELP_COUNT_SERVERS,
+ CLERI_GID_HELP_COUNT_SHARDS,
+ CLERI_GID_HELP_COUNT_USERS,
+ CLERI_GID_HELP_CREATE,
+ CLERI_GID_HELP_CREATE_GROUP,
+ CLERI_GID_HELP_CREATE_USER,
+ CLERI_GID_HELP_DROP,
+ CLERI_GID_HELP_DROP_GROUP,
+ CLERI_GID_HELP_DROP_SERIES,
+ CLERI_GID_HELP_DROP_SERVER,
+ CLERI_GID_HELP_DROP_SHARDS,
+ CLERI_GID_HELP_DROP_USER,
+ CLERI_GID_HELP_FUNCTIONS,
+ CLERI_GID_HELP_GRANT,
+ CLERI_GID_HELP_LIST,
+ CLERI_GID_HELP_LIST_GROUPS,
+ CLERI_GID_HELP_LIST_POOLS,
+ CLERI_GID_HELP_LIST_SERIES,
+ CLERI_GID_HELP_LIST_SERVERS,
+ CLERI_GID_HELP_LIST_SHARDS,
+ CLERI_GID_HELP_LIST_USERS,
+ CLERI_GID_HELP_NOACCESS,
+ CLERI_GID_HELP_REVOKE,
+ CLERI_GID_HELP_SELECT,
+ CLERI_GID_HELP_SHOW,
+ CLERI_GID_HELP_STMT,
+ CLERI_GID_HELP_TIMEIT,
+ CLERI_GID_HELP_TIMEZONES,
+ CLERI_GID_INT_EXPR,
+ CLERI_GID_INT_OPERATOR,
+ CLERI_GID_K_ACCESS,
+ CLERI_GID_K_ACTIVE_HANDLES,
+ CLERI_GID_K_ACTIVE_TASKS,
+ CLERI_GID_K_ADDRESS,
+ CLERI_GID_K_AFTER,
+ CLERI_GID_K_ALL,
+ CLERI_GID_K_ALTER,
+ CLERI_GID_K_AND,
+ CLERI_GID_K_AS,
+ CLERI_GID_K_BACKUP_MODE,
+ CLERI_GID_K_BEFORE,
+ CLERI_GID_K_BETWEEN,
+ CLERI_GID_K_BUFFER_PATH,
+ CLERI_GID_K_BUFFER_SIZE,
+ CLERI_GID_K_COUNT,
+ CLERI_GID_K_CREATE,
+ CLERI_GID_K_CRITICAL,
+ CLERI_GID_K_DATABASE,
+ CLERI_GID_K_DBNAME,
+ CLERI_GID_K_DBPATH,
+ CLERI_GID_K_DEBUG,
+ CLERI_GID_K_DERIVATIVE,
+ CLERI_GID_K_DIFFERENCE,
+ CLERI_GID_K_DROP,
+ CLERI_GID_K_DROP_THRESHOLD,
+ CLERI_GID_K_DURATION_LOG,
+ CLERI_GID_K_DURATION_NUM,
+ CLERI_GID_K_END,
+ CLERI_GID_K_ERROR,
+ CLERI_GID_K_EXPIRATION_LOG,
+ CLERI_GID_K_EXPIRATION_NUM,
+ CLERI_GID_K_EXPRESSION,
+ CLERI_GID_K_FALSE,
+ CLERI_GID_K_FIFO_FILES,
+ CLERI_GID_K_FILTER,
+ CLERI_GID_K_FIRST,
+ CLERI_GID_K_FLOAT,
+ CLERI_GID_K_FOR,
+ CLERI_GID_K_FROM,
+ CLERI_GID_K_FULL,
+ CLERI_GID_K_GRANT,
+ CLERI_GID_K_GROUP,
+ CLERI_GID_K_GROUPS,
+ CLERI_GID_K_HELP,
+ CLERI_GID_K_IDLE_PERCENTAGE,
+ CLERI_GID_K_IDLE_TIME,
+ CLERI_GID_K_IGNORE_THRESHOLD,
+ CLERI_GID_K_INF,
+ CLERI_GID_K_INFO,
+ CLERI_GID_K_INSERT,
+ CLERI_GID_K_INTEGER,
+ CLERI_GID_K_INTERSECTION,
+ CLERI_GID_K_INTERVAL,
+ CLERI_GID_K_IP_SUPPORT,
+ CLERI_GID_K_LAST,
+ CLERI_GID_K_LENGTH,
+ CLERI_GID_K_LIBUV,
+ CLERI_GID_K_LIMIT,
+ CLERI_GID_K_LIST,
+ CLERI_GID_K_LIST_LIMIT,
+ CLERI_GID_K_LOG,
+ CLERI_GID_K_LOG_LEVEL,
+ CLERI_GID_K_MAX,
+ CLERI_GID_K_MAX_OPEN_FILES,
+ CLERI_GID_K_MEAN,
+ CLERI_GID_K_MEDIAN,
+ CLERI_GID_K_MEDIAN_HIGH,
+ CLERI_GID_K_MEDIAN_LOW,
+ CLERI_GID_K_MEM_USAGE,
+ CLERI_GID_K_MERGE,
+ CLERI_GID_K_MIN,
+ CLERI_GID_K_MODIFY,
+ CLERI_GID_K_NAME,
+ CLERI_GID_K_NAN,
+ CLERI_GID_K_NINF,
+ CLERI_GID_K_NOW,
+ CLERI_GID_K_NUMBER,
+ CLERI_GID_K_ONLINE,
+ CLERI_GID_K_OPEN_FILES,
+ CLERI_GID_K_OR,
+ CLERI_GID_K_PASSWORD,
+ CLERI_GID_K_POINTS,
+ CLERI_GID_K_POOL,
+ CLERI_GID_K_POOLS,
+ CLERI_GID_K_PORT,
+ CLERI_GID_K_PREFIX,
+ CLERI_GID_K_PVARIANCE,
+ CLERI_GID_K_READ,
+ CLERI_GID_K_RECEIVED_POINTS,
+ CLERI_GID_K_REINDEX_PROGRESS,
+ CLERI_GID_K_REVOKE,
+ CLERI_GID_K_SELECT,
+ CLERI_GID_K_SELECTED_POINTS,
+ CLERI_GID_K_SELECT_POINTS_LIMIT,
+ CLERI_GID_K_SERIES,
+ CLERI_GID_K_SERVER,
+ CLERI_GID_K_SERVERS,
+ CLERI_GID_K_SET,
+ CLERI_GID_K_SHARDS,
+ CLERI_GID_K_SHARD_DURATION,
+ CLERI_GID_K_SHOW,
+ CLERI_GID_K_SID,
+ CLERI_GID_K_SIZE,
+ CLERI_GID_K_START,
+ CLERI_GID_K_STARTUP_TIME,
+ CLERI_GID_K_STATUS,
+ CLERI_GID_K_STDDEV,
+ CLERI_GID_K_STRING,
+ CLERI_GID_K_SUFFIX,
+ CLERI_GID_K_SUM,
+ CLERI_GID_K_SYMMETRIC_DIFFERENCE,
+ CLERI_GID_K_SYNC_PROGRESS,
+ CLERI_GID_K_TAG,
+ CLERI_GID_K_TAGS,
+ CLERI_GID_K_TEE_PIPE_NAME,
+ CLERI_GID_K_TIMEIT,
+ CLERI_GID_K_TIMEVAL,
+ CLERI_GID_K_TIMEZONE,
+ CLERI_GID_K_TIME_PRECISION,
+ CLERI_GID_K_TO,
+ CLERI_GID_K_TRUE,
+ CLERI_GID_K_TYPE,
+ CLERI_GID_K_UNION,
+ CLERI_GID_K_UNTAG,
+ CLERI_GID_K_UPTIME,
+ CLERI_GID_K_USER,
+ CLERI_GID_K_USERS,
+ CLERI_GID_K_USING,
+ CLERI_GID_K_UUID,
+ CLERI_GID_K_VARIANCE,
+ CLERI_GID_K_VERSION,
+ CLERI_GID_K_WARNING,
+ CLERI_GID_K_WHERE,
+ CLERI_GID_K_WHO_AM_I,
+ CLERI_GID_K_WRITE,
+ CLERI_GID_LIMIT_EXPR,
+ CLERI_GID_LIST_GROUPS,
+ CLERI_GID_LIST_POOLS,
+ CLERI_GID_LIST_SERIES,
+ CLERI_GID_LIST_SERVERS,
+ CLERI_GID_LIST_SHARDS,
+ CLERI_GID_LIST_STMT,
+ CLERI_GID_LIST_TAGS,
+ CLERI_GID_LIST_USERS,
+ CLERI_GID_LOG_KEYWORDS,
+ CLERI_GID_MERGE_AS,
+ CLERI_GID_POOL_COLUMNS,
+ CLERI_GID_POOL_PROPS,
+ CLERI_GID_PREFIX_EXPR,
+ CLERI_GID_REVOKE_STMT,
+ CLERI_GID_REVOKE_USER,
+ CLERI_GID_R_COMMENT,
+ CLERI_GID_R_DOUBLEQ_STR,
+ CLERI_GID_R_FLOAT,
+ CLERI_GID_R_GRAVE_STR,
+ CLERI_GID_R_INTEGER,
+ CLERI_GID_R_REGEX,
+ CLERI_GID_R_SINGLEQ_STR,
+ CLERI_GID_R_TIME_STR,
+ CLERI_GID_R_UINTEGER,
+ CLERI_GID_R_UUID_STR,
+ CLERI_GID_SELECT_AGGREGATE,
+ CLERI_GID_SELECT_AGGREGATES,
+ CLERI_GID_SELECT_STMT,
+ CLERI_GID_SERIES_ALL,
+ CLERI_GID_SERIES_COLUMNS,
+ CLERI_GID_SERIES_MATCH,
+ CLERI_GID_SERIES_NAME,
+ CLERI_GID_SERIES_PARENTHESES,
+ CLERI_GID_SERIES_RE,
+ CLERI_GID_SERIES_SETOPR,
+ CLERI_GID_SERVER_COLUMNS,
+ CLERI_GID_SET_ADDRESS,
+ CLERI_GID_SET_BACKUP_MODE,
+ CLERI_GID_SET_DROP_THRESHOLD,
+ CLERI_GID_SET_EXPIRATION_LOG,
+ CLERI_GID_SET_EXPIRATION_NUM,
+ CLERI_GID_SET_EXPRESSION,
+ CLERI_GID_SET_IGNORE_THRESHOLD,
+ CLERI_GID_SET_LIST_LIMIT,
+ CLERI_GID_SET_LOG_LEVEL,
+ CLERI_GID_SET_NAME,
+ CLERI_GID_SET_PASSWORD,
+ CLERI_GID_SET_PORT,
+ CLERI_GID_SET_SELECT_POINTS_LIMIT,
+ CLERI_GID_SET_TEE_PIPE_NAME,
+ CLERI_GID_SET_TIMEZONE,
+ CLERI_GID_SHARD_COLUMNS,
+ CLERI_GID_SHOW_STMT,
+ CLERI_GID_START,
+ CLERI_GID_STRING,
+ CLERI_GID_STR_OPERATOR,
+ CLERI_GID_SUFFIX_EXPR,
+ CLERI_GID_TAG_COLUMNS,
+ CLERI_GID_TAG_NAME,
+ CLERI_GID_TAG_SERIES,
+ CLERI_GID_TIMEIT_STMT,
+ CLERI_GID_TIME_EXPR,
+ CLERI_GID_UNTAG_SERIES,
+ CLERI_GID_USER_COLUMNS,
+ CLERI_GID_UUID,
+ CLERI_GID_WHERE_GROUP,
+ CLERI_GID_WHERE_POOL,
+ CLERI_GID_WHERE_SERIES,
+ CLERI_GID_WHERE_SERVER,
+ CLERI_GID_WHERE_SHARD,
+ CLERI_GID_WHERE_TAG,
+ CLERI_GID_WHERE_USER,
+ CLERI_GID__BOOLEAN,
+ CLERI_END // can be used to get the enum length
+};
+
+#endif /* CLERI_EXPORT_SIRI_GRAMMAR_GRAMMAR_H_ */
+
--- /dev/null
+/*
+ * gramp.h - SiriDB Grammar Properties.
+ *
+ * Note: we need this file up-to-date with the grammar. The grammar has
+ * keywords starting with K_ so the will all be sorted.
+ * KW_OFFSET should be set to the first keyword and KW_COUNT needs the
+ * last keyword in the grammar.
+ *
+ */
+#ifndef SIRI_GRAMP_H_
+#define SIRI_GRAMP_H_
+
+#include <siri/grammar/grammar.h>
+#include <siri/inc.h>
+
+/* keywords */
+#define KW_OFFSET CLERI_GID_K_ACCESS
+#define KW_COUNT CLERI_GID_K_WRITE + 1 - KW_OFFSET
+
+/* aggregation functions */
+#define F_OFFSET CLERI_GID_F_COUNT
+
+/* help statements */
+#define HELP_OFFSET CLERI_GID_HELP_ACCESS
+#define HELP_COUNT CLERI_GID_HELP_TIMEZONES + 1 - HELP_OFFSET
+
+
+
+#if CLERI_VERSION_MINOR >= 12
+#if SIRIDB_IS64BIT
+#define CLERI_NODE_DATA(__node) ((int64_t)(__node)->data)
+#define CLERI_NODE_DATA_ADDR(__node) ((int64_t *) &(__node)->data)
+#else
+#define CLERI_NODE_DATA(__node) *((int64_t *)(__node)->data)
+#define CLERI_NODE_DATA_ADDR(__node) ((int64_t *)(__node)->data)
+#endif
+#else
+#define CLERI_NODE_DATA(__node) (__node)->result
+#define CLERI_NODE_DATA_ADDR(__node) &(__node)->result
+#endif
+
+#endif /* SIRI_GRAMP_H_ */
--- /dev/null
+/*
+ * health.h - SiriDB Health Status.
+ */
+#ifndef SIRI_HEALTH_H_
+#define SIRI_HEALTH_H_
+
+#include <lib/http_parser.h>
+#include <uv.h>
+
+#define SIRIDB_HEALTH_FLAG 1<<30
+
+typedef struct siri_health_request_s siri_health_request_t;
+
+int siri_health_init(void);
+void siri_health_close(siri_health_request_t * web_request);
+static inline _Bool siri_health_is_handle(uv_handle_t * handle);
+
+struct siri_health_request_s
+{
+ uint32_t flags; /* maps to sirnet_stream_t tp for cleanup */
+ uint32_t pad0_;
+ _Bool is_closed;
+ uv_write_t req;
+ uv_stream_t uvstream;
+ http_parser parser;
+ uv_buf_t * response;
+};
+
+static inline _Bool siri_health_is_handle(uv_handle_t * handle)
+{
+ return
+ handle->type == UV_TCP &&
+ handle->data &&
+ (((siri_health_request_t *) handle->data)->flags & SIRIDB_HEALTH_FLAG);
+}
+
+#endif /* TI_HEALTH_H_ */
--- /dev/null
+/*
+ * heartbeat.h - Heart-beat task SiriDB.
+ *
+ * There is one and only one heart-beat task thread running for SiriDB. For
+ * this reason we do not need to parse data but we should only take care for
+ * locks while writing data.
+ */
+#ifndef SIRI_HEARTBEAT_H_
+#define SIRI_HEARTBEAT_H_
+
+#include <siri/siri.h>
+
+void siri_heartbeat_init(siri_t * siri);
+void siri_heartbeat_stop(siri_t * siri);
+void siri_heartbeat_force(void);
+
+#endif /* SIRI_HEARTBEAT_H_ */
--- /dev/null
+/*
+ * help.h - Help for SiriDB.
+ */
+#ifndef SIRI_HELP_H_
+#define SIRI_HELP_H_
+
+#include <siri/grammar/gramp.h>
+
+const char * siri_help_get(
+ uint16_t gid,
+ const char * help_name,
+ char * err_msg);
+
+void siri_help_free(void);
+
+#endif /* SIRI_HELP_H_ */
--- /dev/null
+#if UINTPTR_MAX == 0xffffffff
+#define SIRIDB_IS64BIT 0
+#elif UINTPTR_MAX == 0xffffffffffffffff
+#define SIRIDB_IS64BIT 1
+#else
+#define SIRIDB_IS64BIT __WORDSIZE == 64
+#endif
+
+
+#if SIRIDB_IS64BIT
+#define SIRIDB_EXPR_ALLOC 0
+#else
+#define SIRIDB_EXPR_ALLOC CLERI_VERSION_MINOR >= 12
+#endif
--- /dev/null
+/*
+ * bserver.h - Listen to back-end SiriDB Server.
+ */
+#ifndef SIRINET_BSERVER_H_
+#define SIRINET_BSERVER_H_
+
+#include <siri/siri.h>
+
+int sirinet_bserver_init(siri_t * siri);
+
+#endif /* SIRINET_BSERVER_H_ */
--- /dev/null
+/*
+ * clserver.h - Listen for client requests.
+ */
+#ifndef SIRINET_CLSERVER_H_
+#define SIRINET_CLSERVER_H_
+
+#include <uv.h>
+#include <siri/siri.h>
+
+typedef ssize_t (*sirinet_clserver_getfile)(char ** buffer, siridb_t * siridb);
+
+int sirinet_clserver_init(siri_t * siri);
+
+#endif /* SIRINET_CLSERVER_H_ */
--- /dev/null
+/*
+ * pipe.h - Named Pipe support.
+ */
+#ifndef SIRINET_PIPE_H_
+#define SIRINET_PIPE_H_
+
+#include <uv.h>
+
+char * sirinet_pipe_name(uv_pipe_t * client);
+void sirinet_pipe_unlink(uv_pipe_t * client);
+
+#endif /* SIRINET_PIPE_H_ */
--- /dev/null
+/*
+ * pkg.h - SiriDB Package type.
+ */
+#ifndef SIRINET_PKG_H_
+#define SIRINET_PKG_H_
+
+typedef struct sirinet_pkg_s sirinet_pkg_t;
+
+#include <inttypes.h>
+#include <qpack/qpack.h>
+#include <siri/net/stream.h>
+#include <uv.h>
+
+sirinet_pkg_t * sirinet_pkg_new(
+ uint16_t pid,
+ uint32_t len,
+ uint8_t tp,
+ const unsigned char * data);
+qp_packer_t * sirinet_packer_new(size_t alloc_size);
+sirinet_pkg_t * sirinet_packer2pkg(
+ qp_packer_t * packer,
+ uint16_t pid,
+ uint8_t tp);
+sirinet_pkg_t * sirinet_pkg_err(
+ uint16_t pid,
+ uint32_t len,
+ uint8_t tp,
+ const char * msg);
+
+int sirinet_pkg_send(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+sirinet_pkg_t * sirinet_pkg_dup(sirinet_pkg_t * pkg);
+
+/* Shortcut to print an packer object */
+#define sn_packer_print(packer) \
+ qp_print(packer->buffer + sizeof(sirinet_pkg_t), packer->len - sizeof(sirinet_pkg_t))
+
+struct sirinet_pkg_s
+{
+ uint32_t len; /* length of data, sizeof(sirinet_pkg_t) is not included */
+ uint16_t pid;
+ uint8_t tp;
+ uint8_t checkbit;
+ unsigned char data[];
+};
+
+#endif /* SIRINET_PKG_H_ */
--- /dev/null
+/*
+ * promise.c - Promise used for sending to data to SiriDB servers.
+ */
+#ifndef SIRINET_PROMISE_H_
+#define SIRINET_PROMISE_H_
+
+#define PROMISE_DEFAULT_TIMEOUT 30000 /* 30 seconds */
+
+typedef enum
+{
+ PROMISE_TIMEOUT_ERROR=-4, /* in case of a time out */
+ PROMISE_WRITE_ERROR, /* socket write error */
+ PROMISE_CANCELLED_ERROR, /* timer is cancelled */
+ PROMISE_PKG_TYPE_ERROR, /* unexpected package type received */
+ PROMISE_SUCCESS=0
+} sirinet_promise_status_t;
+
+typedef struct sirinet_promise_s sirinet_promise_t;
+typedef void (* sirinet_promise_cb)(
+ sirinet_promise_t * promise,
+ void * data,
+ int status);
+
+#include <uv.h>
+#include <siri/net/stream.h>
+#include <siri/db/server.h>
+#include <siri/net/pkg.h>
+
+
+const char * sirinet_promise_strstatus(sirinet_promise_status_t status);
+
+#define sirinet_promise_incref(p__) (p__)->ref++
+#define sirinet_promise_decref(p__) if (!--(p__)->ref) free(p__)
+
+/* the callback will always be called and is responsible to free the promise */
+struct sirinet_promise_s
+{
+ uint16_t pid;
+ uint16_t ref;
+ uv_timer_t * timer;
+ sirinet_promise_cb cb;
+ siridb_server_t * server;
+ sirinet_pkg_t * pkg;
+ void * data;
+};
+
+#endif /* SIRINET_PROMISE_H_ */
--- /dev/null
+/*
+ * promises.h - Collection for promised.
+ */
+#ifndef SIRINET_PROMISES_H_
+#define SIRINET_PROMISES_H_
+
+typedef struct sirinet_promises_s sirinet_promises_t;
+
+#include <vec/vec.h>
+typedef void (* sirinet_promises_cb)(
+ vec_t * promises,
+ void * data);
+
+#include <siri/net/promise.h>
+#include <siri/net/pkg.h>
+
+sirinet_promises_t * sirinet_promises_new(
+ size_t size,
+ sirinet_promises_cb cb,
+ void * data,
+ sirinet_pkg_t * pkg);
+void sirinet_promises_llist_free(vec_t * promises);
+void sirinet_promises_on_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+
+#define SIRINET_PROMISES_CHECK(promises) \
+if (promises->promises->len == promises->promises->size) \
+{ \
+ free(promises->pkg); \
+ promises->cb(promises->promises, promises->data); \
+ vec_free(promises->promises); \
+ free(promises); \
+}
+
+struct sirinet_promises_s
+{
+ sirinet_promises_cb cb;
+ vec_t * promises;
+ void * data;
+ sirinet_pkg_t * pkg;
+};
+
+#endif /* SIRINET_PROMISES_H_ */
--- /dev/null
+/*
+ * protocol.c - Protocol definitions for SiriDB.
+ */
+#ifndef SIRINET_PROTOCOL_H_
+#define SIRINET_PROTOCOL_H_
+
+typedef enum
+{
+ /* Public requests */
+ CPROTO_REQ_QUERY=0, /* (query, time_precision) */
+ CPROTO_REQ_INSERT=1, /* series with points map/array */
+ CPROTO_REQ_AUTH=2, /* (user, password, dbname) */
+ CPROTO_REQ_PING=3, /* empty */
+
+ /* Internal usage only */
+ CPROTO_REQ_REGISTER_SERVER=6, /* (uuid, host, port, pool) */
+ CPROTO_REQ_FILE_SERVERS=7, /* empty */
+ CPROTO_REQ_FILE_USERS=8, /* empty */
+ CPROTO_REQ_FILE_GROUPS=9, /* empty */
+ CPROTO_REQ_FILE_DATABASE=10, /* empty */
+
+ /* Public Service API request */
+ CPROTO_REQ_SERVICE=32, /* (user, password, request, {...}) */
+} cproto_client_t;
+
+typedef enum
+{
+ /* success */
+ CPROTO_RES_QUERY=0, /* {query response data} */
+ CPROTO_RES_INSERT=1, /* {"success_msg": ...} */
+ CPROTO_RES_AUTH_SUCCESS=2, /* empty */
+ CPROTO_RES_ACK=3, /* empty */
+ CPROTO_RES_FILE=5, /* file content */
+
+ /* Service API success */
+ CPROTO_ACK_SERVICE=32, /* empty */
+ CPROTO_ACK_SERVICE_DATA=33, /* [...] */
+
+ /* errors 64-69 are errors with messages */
+ CPROTO_ERR_MSG=64, /* {"error_msg": ...} */
+ CPROTO_ERR_QUERY=65, /* {"error_msg": ...} */
+ CPROTO_ERR_INSERT=66, /* {"error_msg": ...} */
+ CPROTO_ERR_SERVER=67, /* {"error_msg": ...} */
+ CPROTO_ERR_POOL=68, /* {"error_msg": ...} */
+ CPROTO_ERR_USER_ACCESS=69, /* {"error_msg": ...} */
+ CPROTO_ERR=70, /* empty (use for unexpected errors)*/
+ CPROTO_ERR_NOT_AUTHENTICATED=71, /* empty */
+ CPROTO_ERR_AUTH_CREDENTIALS=72, /* empty */
+ CPROTO_ERR_AUTH_UNKNOWN_DB=73, /* empty */
+ CPROTO_ERR_FILE=75, /* empty */
+
+ /* Service API errors */
+ CPROTO_ERR_SERVICE=96, /* {"error_msg": ...} */
+ CPROTO_ERR_SERVICE_INVALID_REQUEST=97,/* empty */
+ CPROTO_DEFERRED=127 /* deferred... */
+} cproto_server_t;
+
+typedef enum
+{
+ BPROTO_AUTH_REQUEST=128, /* (uuid, dbname, flags, version,
+ min_version, dbpath, buffer_path,
+ buffer_size, startup_time,
+ address, port) */
+ BPROTO_FLAGS_UPDATE, /* flags */
+ BPROTO_LOG_LEVEL_UPDATE, /* log_level */
+ BPROTO_REPL_FINISHED, /* empty */
+ BPROTO_QUERY_SERVER, /* (query, time_precision) */
+ BPROTO_QUERY_UPDATE, /* (query, time_precision) */
+ BPROTO_INSERT_POOL, /* {series: points, ...} */
+ BPROTO_INSERT_SERVER, /* {series: points, ...} */
+ BPROTO_INSERT_TEST_POOL, /* {series: points, ...} */
+ BPROTO_INSERT_TEST_SERVER, /* {series: points, ...} */
+ BPROTO_INSERT_TESTED_POOL, /* {series: points, ...} */
+ BPROTO_INSERT_TESTED_SERVER, /* {series: points, ...} */
+ BPROTO_REGISTER_SERVER, /* (uuid, host, port, pool) */
+ BPROTO_DROP_SERIES, /* series_name */
+ BPROTO_REQ_GROUPS, /* empty */
+ BPROTO_ENABLE_BACKUP_MODE, /* empty */
+ BPROTO_DISABLE_BACKUP_MODE, /* empty */
+ BPROTO_TEE_PIPE_NAME_UPDATE, /* tee pipe name */
+ BPROTO_DROP_DATABASE, /* empty */
+ BPROTO_REQ_TAGS, /* empty */
+ BPROTO_SERIES_TAGS, /* [series name, tag name, ...] */
+ BPROTO_EMPTY_TAGS, /* [tag name, tag name, ...] */
+} bproto_client_t;
+
+/*
+ * Success and error messages are set this way so that we have one range
+ * for error messages between 64.. 191.
+ *
+ * Client Success messages in range 0..63
+ * Client Error messages in range 64..126 (127 is used as deferred)
+ * Back-end Error messages in range 128..191
+ * Back-end Success messages in range 192..254 (exclude 255)
+ */
+typedef enum
+{
+ /* Mappings to client protocol messages */
+ /* success */
+ BPROTO_RES_QUERY=CPROTO_RES_QUERY, /* {query response data} */
+
+ /* errors */
+ BPROTO_ERR_QUERY=CPROTO_ERR_QUERY, /* {"error_msg": ...} */
+ BPROTO_ERR_SERVER=CPROTO_ERR_SERVER, /* {"error_msg": ...} */
+ BPROTO_ERR_POOL=CPROTO_ERR_POOL, /* {"error_msg": ...} */
+
+ /* Back-end specific protocol messages */
+ /* errors */
+ BPROTO_AUTH_ERR_UNKNOWN_UUID=128, /* empty */
+ BPROTO_AUTH_ERR_UNKNOWN_DBNAME, /* empty */
+ BPROTO_AUTH_ERR_INVALID_UUID, /* empty */
+ BPROTO_AUTH_ERR_VERSION_TOO_OLD, /* empty */
+ BPROTO_AUTH_ERR_VERSION_TOO_NEW, /* empty */
+ BPROTO_ERR_NOT_AUTHENTICATED, /* empty */
+ BPROTO_ERR_INSERT, /* empty */
+ BPROTO_ERR_REGISTER_SERVER, /* empty */
+ BPROTO_ERR_DROP_SERIES, /* empty */
+ BPROTO_ERR_ENABLE_BACKUP_MODE, /* empty */
+ BPROTO_ERR_DISABLE_BACKUP_MODE, /* empty */
+
+ /* success */
+ BPROTO_AUTH_SUCCESS=192, /* empty */
+ BPROTO_ACK_FLAGS, /* empty */
+ BPROTO_ACK_LOG_LEVEL, /* empty */
+ BPROTO_ACK_INSERT, /* empty */
+ BPROTO_ACK_REPL_FINISHED, /* empty */
+ BPROTO_ACK_REGISTER_SERVER, /* empty */
+ BPROTO_ACK_DROP_SERIES, /* empty */
+ BPROTO_ACK_ENABLE_BACKUP_MODE, /* empty */
+ BPROTO_ACK_DISABLE_BACKUP_MODE, /* empty */
+ BPROTO_RES_GROUPS, /* [[name, series], ...] */
+ BPROTO_ACK_TEE_PIPE_NAME, /* empty */
+ BPROTO_ACK_DROP_DATABASE, /* empty */
+ BPROTO_RES_TAGS, /* [[name, series], ...] */
+ BPROTO_ACK_SERIES_TAGS, /* empty */
+ BPROTO_ACK_EMPTY_TAGS, /* empty */
+} bproto_server_t;
+
+#define sirinet_protocol_is_error(tp) (tp >= 64 && tp < 192)
+#define sirinet_protocol_is_error_msg(tp) (tp >= 64 && tp < 70)
+
+const char * sirinet_cproto_client_str(cproto_client_t n);
+const char * sirinet_cproto_server_str(cproto_server_t n);
+const char * sirinet_bproto_client_str(bproto_client_t n);
+const char * sirinet_bproto_server_str(bproto_server_t n);
+
+#endif /* SIRINET_PROTOCOL_H_ */
--- /dev/null
+/*
+ * stream.h - For handling streams.
+ */
+#ifndef SIRINET_STREAM_H_
+#define SIRINET_STREAM_H_
+
+#define RESET_BUF_SIZE 2097152 /* 2 MB */
+
+typedef enum
+{
+ STREAM_TCP_CLIENT,
+ STREAM_TCP_BACKEND,
+ STREAM_TCP_SERVER,
+ STREAM_TCP_MANAGE,
+ STREAM_PIPE_CLIENT,
+ STREAM_API_CLIENT,
+} sirinet_stream_tp_t;
+
+typedef struct sirinet_stream_s sirinet_stream_t;
+
+#include <uv.h>
+#include <siri/db/db.h>
+#include <siri/net/pkg.h>
+
+typedef void (* on_data_cb_t)(sirinet_stream_t * stream, sirinet_pkg_t * pkg);
+
+sirinet_stream_t * sirinet_stream_new(sirinet_stream_tp_t tp, on_data_cb_t cb);
+char * sirinet_stream_name(sirinet_stream_t * client);
+void sirinet_stream_alloc_buffer(
+ uv_handle_t * handle,
+ size_t suggested_size,
+ uv_buf_t * buf);
+void sirinet_stream_on_data(
+ uv_stream_t * client,
+ ssize_t nread,
+ const uv_buf_t * buf);
+void sirinet__stream_free(uv_stream_t * uvclient);
+
+#define sirinet_stream_incref(client__) \
+ __atomic_add_fetch(&(client__)->ref, 1, __ATOMIC_SEQ_CST)
+
+#define sirinet_stream_decref(client__) \
+ if (!__atomic_sub_fetch(&(client__)->ref, 1, __ATOMIC_SEQ_CST)) uv_close( \
+ (uv_handle_t *) (client__)->stream, \
+ (uv_close_cb) sirinet__stream_free)
+
+struct sirinet_stream_s
+{
+ uint32_t tp; /* maps to siridb_tee_t flags for cleanup */
+ uint32_t ref;
+ on_data_cb_t on_data;
+ siridb_t * siridb;
+ void * origin; /* can be a user, server or NULL */
+ char * buf;
+ size_t len;
+ size_t size;
+ uv_stream_t * stream;
+};
+
+#endif /* SIRINET_STREAM_H_ */
--- /dev/null
+/*
+ * tcp.h - TCP support.
+ */
+#ifndef SIRINET_TCP_H_
+#define SIRINET_TCP_H_
+
+#include <uv.h>
+
+/* Warning: do not change the order! (maps to dns_req_family_map) */
+enum
+{
+ IP_SUPPORT_ALL,
+ IP_SUPPORT_IPV4ONLY,
+ IP_SUPPORT_IPV6ONLY
+};
+
+/* dns_req_family_map maps to IP_SUPPORT values defined in socket.h */
+static int DNS_REQ_FAMILY_MAP[3] = {AF_UNSPEC, AF_INET, AF_INET6};
+
+const char * sirinet_tcp_ip_support_str(uint8_t ip_support);
+char * sirinet_tcp_name(uv_tcp_t * client);
+int sirinet_extract_addr_port(
+ char * s,
+ char * addr,
+ uint16_t * port);
+
+static inline int dns_req_family_map(int i)
+{
+ return DNS_REQ_FAMILY_MAP[i];
+}
+
+#endif /* SIRINET_TCP_H_ */
--- /dev/null
+/*
+ * optimize.h - Optimize task SiriDB.
+ *
+ * There is one and only one optimize task thread running for SiriDB. For this
+ * reason we do not need to parse data but we should only take care for locks
+ * while writing data.
+ *
+ * Thread debugging:
+ * log_debug("getpid: %d - pthread_self: %lu",getpid(), pthread_self());
+ *
+ */
+#ifndef SIRI_OPTIMIZE_H_
+#define SIRI_OPTIMIZE_H_
+
+#define SIRI_OPTIMIZE_PENDING 0
+#define SIRI_OPTIMIZE_RUNNING 1
+#define SIRI_OPTIMIZE_CANCELLED 2
+#define SIRI_OPTIMIZE_PAUSED 3 /* only set in 'siri_optimize_wait' */
+#define SIRI_OPTIMIZE_PAUSED_MAIN 4
+
+typedef struct siri_optimize_s siri_optimize_t;
+
+#define SIRI_OPTIMZE_IS_PAUSED (siri.optimize->status >= SIRI_OPTIMIZE_PAUSED)
+
+#include <uv.h>
+#include <stdio.h>
+#include <siri/siri.h>
+
+void siri_optimize_init(siri_t * siri);
+void siri_optimize_stop(void);
+void siri_optimize_pause(void);
+void siri_optimize_continue(void);
+int siri_optimize_wait(void);
+int siri_optimize_create_idx(const char * fn);
+int siri_optimize_finish_idx(const char * fn, int remove_old);
+
+struct siri_optimize_s
+{
+ uv_timer_t timer;
+ int status;
+ time_t start;
+ uv_work_t work;
+ uint16_t pause;
+ FILE * idx_fp;
+ char * idx_fn;
+};
+#endif /* SIRI_OPTIMIZE_H_ */
--- /dev/null
+/*
+ * account.h - SiriDB Service Account.
+ */
+#ifndef SIRI_SERVICE_ACCOUNT_H_
+#define SIRI_SERVICE_ACCOUNT_H_
+
+typedef struct siri_service_account_s siri_service_account_t;
+
+#include <qpack/qpack.h>
+#include <siri/siri.h>
+
+int siri_service_account_init(siri_t * siri);
+void siri_service_account_destroy(siri_t * siri);
+int siri_service_account_new(
+ siri_t * siri,
+ qp_obj_t * qp_account,
+ qp_obj_t * qp_password,
+ int is_encrypted,
+ char * err_msg);
+int siri_service_account_check(
+ siri_t * siri,
+ qp_obj_t * qp_account,
+ qp_obj_t * qp_password,
+ char * err_msg);
+int siri_service_account_change_password(
+ siri_t * siri,
+ qp_obj_t * qp_account,
+ qp_obj_t * qp_password,
+ char * err_msg);
+_Bool siri_service_account_check_basic(
+ siri_t * siri,
+ const char * data,
+ size_t n);
+int siri_service_account_drop(
+ siri_t * siri,
+ qp_obj_t * qp_account,
+ char * err_msg);
+int siri_service_account_save(siri_t * siri, char * err_msg);
+
+struct siri_service_account_s
+{
+ char * account;
+ char * password; /* keeps an encrypted password */
+};
+
+#endif /* SIRI_SERVICE_ACCOUNT_H_ */
--- /dev/null
+/*
+ * client.h - Client for expanding a SiriDB database.
+ */
+#ifndef SIRI_SERVICE_CLIENT_H_
+#define SIRI_SERVICE_CLIENT_H_
+
+typedef struct siri_service_client_s siri_service_client_t;
+
+#include <inttypes.h>
+#include <uv.h>
+#include <qpack/qpack.h>
+#include <uuid/uuid.h>
+#include <siri/net/pkg.h>
+
+int siri_service_client_request(
+ uint16_t pid,
+ uint16_t port,
+ int pool,
+ uuid_t * uuid,
+ qp_obj_t * host,
+ qp_obj_t * username,
+ qp_obj_t * password,
+ qp_obj_t * dbname,
+ const char * dbpath,
+ sirinet_stream_t * client,
+ char * err_msg);
+
+void siri_service_client_free(siri_service_client_t * adm_client);
+
+struct siri_service_client_s
+{
+ uint8_t request;
+ uint8_t flags;
+ uint16_t pid;
+ uint16_t port;
+ uuid_t uuid;
+ int pool; /* -1 for a new pool */
+ char * host;
+ char * username;
+ char * password;
+ char * dbname;
+ char * dbpath;
+ sirinet_stream_t * client;
+ sirinet_pkg_t * pkg;
+};
+
+#endif /* SIRI_SERVICE_CLIENT_H_ */
--- /dev/null
+/*
+ * request.h - SiriDB Service Request.
+ */
+#ifndef SIRI_SERVICE_REQUEST_H_
+#define SIRI_SERVICE_REQUEST_H_
+
+typedef enum
+{
+ SERVICE_NEW_ACCOUNT,
+ SERVICE_CHANGE_PASSWORD,
+ SERVICE_DROP_ACCOUNT,
+ SERVICE_NEW_DATABASE,
+ SERVICE_NEW_POOL,
+ SERVICE_NEW_REPLICA,
+ SERVICE_DROP_DATABASE,
+ SERVICE_GET_VERSION=64,
+ SERVICE_GET_ACCOUNTS,
+ SERVICE_GET_DATABASES
+} service_request_t;
+
+#include <qpack/qpack.h>
+#include <siri/net/protocol.h>
+
+int siri_service_request_init(void);
+void siri_service_request_destroy(void);
+cproto_server_t siri_service_request(
+ int tp,
+ qp_unpacker_t * qp_unpacker,
+ qp_packer_t ** packaddr,
+ uint16_t pid,
+ sirinet_stream_t * client,
+ char * err_msg);
+void siri_service_request_rollback(const char * dbpath);
+
+
+#endif /* SIRI_SERVICE_REQUEST_H_ */
--- /dev/null
+/*
+ * siri.h - Root for SiriDB.
+ *
+ *
+ * Info siri->siridb_mutex:
+ *
+ * Main thread:
+ * siri->siridb_list : read (no lock) write (lock)
+ *
+ * Other threads:
+ * siri->siridb_list : read (lock) write (not allowed)
+ *
+ */
+#ifndef SIRI_H_
+#define SIRI_H_
+
+#define PCRE2_CODE_UNIT_WIDTH 8
+
+#define SIRI_MAX_SIZE_ERR_MSG 1024
+#define MAX_NUMBER_DB 1024
+
+#if defined(__GLIBC__)
+#define strerror_si(__err, __buf, __sz) \
+ strerror_r(__err, __buf, __sz)
+#else
+#define strerror_si(__err, __buf, __sz) \
+ (strerror_r(__err, __buf, __sz) == 0 ? __buf : "unexpected error")
+#endif
+
+typedef enum
+{
+ SIRI_STATUS_LOADING,
+ SIRI_STATUS_RUNNING,
+ SIRI_STATUS_CLOSING
+} siri_status_t;
+
+typedef struct siri_s siri_t;
+
+extern siri_t siri;
+
+#include <uv.h>
+#include <pcre2.h>
+#include <siri/grammar/grammar.h>
+#include <siri/db/db.h>
+#include <siri/file/handler.h>
+#include <stdbool.h>
+#include <siri/optimize.h>
+#include <siri/backup.h>
+#include <siri/heartbeat.h>
+#include <siri/cfg/cfg.h>
+#include <siri/args/args.h>
+#include <llist/llist.h>
+
+void siri_setup_logger(void);
+int siri_start(void);
+void siri_free(void);
+int make_database_directory(void);
+void set_max_open_files_limit(void);
+
+struct siri_s
+{
+ siri_status_t status;
+ uv_loop_t * loop;
+ cleri_grammar_t * grammar;
+ llist_t * siridb_list;
+ siri_fh_t * fh;
+ siri_optimize_t * optimize;
+ uv_timer_t * backup;
+ uv_timer_t * heartbeat;
+ uv_timer_t * buffersync;
+ siri_cfg_t * cfg;
+ siri_args_t * args;
+ uv_mutex_t siridb_mutex;
+ uint32_t startup_time;
+
+ /* initialized by sidi_service_account_init */
+ llist_t * accounts;
+
+ /* initialized by sidi_service_request_init */
+ pcre2_code * dbname_regex;
+ pcre2_match_data * dbname_match_data;
+
+ /* socket and promises used for expanding (client) */
+ sirinet_stream_t * client;
+ uv_timer_t timer;
+};
+
+#endif /* SIRI_H_ */
--- /dev/null
+/*
+ * version.h - SiriDB version info.
+ */
+#ifndef SIRI_VERSION_H_
+#define SIRI_VERSION_H_
+
+#define SIRIDB_VERSION_MAJOR 2
+#define SIRIDB_VERSION_MINOR 0
+#define SIRIDB_VERSION_PATCH 43
+
+/*
+ * Use SIRIDB_VERSION_PRE_RELEASE for alpha release versions.
+ * This should be an empty string when building a final release.
+ * Examples: "-alpha-0" "-alpha-1", ""
+ * Note that debian alpha packages should use versions like this:
+ * 2.0.34-0alpha0
+ */
+#define SIRIDB_VERSION_PRE_RELEASE ""
+
+#ifndef NDEBUG
+#define SIRIDB_VERSION_BUILD_RELEASE "+debug"
+#else
+#define SIRIDB_VERSION_BUILD_RELEASE ""
+#endif
+
+#define SIRIDB_STRINGIFY(num) #num
+#define SIRIDB_VERSION_STR(major,minor,patch) \
+ SIRIDB_STRINGIFY(major) "." \
+ SIRIDB_STRINGIFY(minor) "." \
+ SIRIDB_STRINGIFY(patch)
+
+#define SIRIDB_VERSION SIRIDB_VERSION_STR( \
+ SIRIDB_VERSION_MAJOR, \
+ SIRIDB_VERSION_MINOR, \
+ SIRIDB_VERSION_PATCH) \
+ SIRIDB_VERSION_PRE_RELEASE \
+ SIRIDB_VERSION_BUILD_RELEASE
+
+#define SIRIDB_CONTRIBUTERS \
+ "https://github.com/SiriDB/siridb-server/contributors"
+#define SIRIDB_HOME_PAGE \
+ "https://siridb.net"
+
+int siri_version_cmp(const char * version_a, const char * version_b);
+
+/* SiriDB can only connect with servers having at least this version. */
+#define SIRIDB_MINIMAL_VERSION "2.0.0"
+
+#endif /* SIRI_VERSION_H_ */
--- /dev/null
+/*
+ * timeit.h - Timeit.
+ */
+#ifndef TIMEIT_H_
+#define TIMEIT_H_
+
+#include <time.h>
+
+double timeit_get(struct timespec * start);
+
+#define timeit_start(start) clock_gettime(CLOCK_MONOTONIC, start)
+
+/*
+ * Usage:
+ *
+ * struct timespec start;
+ * timeit_start(&start);
+ *
+ * ... some code ....
+ *
+ * log_debug("Time in milliseconds: %f",timeit_stop(&start));
+ */
+
+#endif /* TIMEIT_H_ */
--- /dev/null
+/*
+ * vec.h - Vector List.
+ */
+#ifndef VEC_H_
+#define VEC_H_
+
+#define VEC_DEFAULT_SIZE 8
+
+typedef struct vec_s vec_t;
+typedef struct vec_object_s vec_object_t;
+typedef void (*vec_destroy_cb)(void *);
+
+#include <stdio.h>
+#include <stddef.h>
+#include <inttypes.h>
+
+vec_t * vec_new(size_t size);
+void vec_destroy(vec_t * vec, vec_destroy_cb cb);
+vec_t * vec_copy(vec_t * source);
+void vec_compact(vec_t ** vec);
+int vec_append_safe(vec_t ** vec, void * data);
+
+/*
+ * Expects the object to have a object->ref (uint_xxx_t) on top of the
+ * objects definition.
+ */
+#define vec_object_incref(object__) __atomic_add_fetch(&((vec_object_t * ) object__)->ref, 1, __ATOMIC_SEQ_CST)
+
+/*
+ * Expects the object to have a object->ref (uint_xxx_t) on top of the
+ * objects definition.
+ *
+ * WARNING: Be careful using 'vec_object_decref' only when being sure
+ * there are still references left on the object since an object
+ * probably needs specific cleanup tasks.
+ */
+#define vec_object_decref(object__) __atomic_sub_fetch(&((vec_object_t * ) object__)->ref, 1, __ATOMIC_SEQ_CST)
+
+/*
+ * Append data to the list. This functions assumes the list can hold the new
+ * data is therefore not safe.
+ */
+#define vec_append(vec, _data) vec->data[vec->len++] = _data
+
+/*
+ * Destroy the vector.
+ */
+#define vec_free(vec) free(vec)
+
+/*
+ * Pop the last item from the list
+ */
+#define vec_pop(vec) vec->data[--vec->len]
+
+struct vec_object_s
+{
+ uint32_t ref;
+};
+
+struct vec_s
+{
+ size_t size;
+ size_t len;
+ void * data[];
+};
+
+#endif /* VEC_H_ */
--- /dev/null
+/*
+ * xmath.h - Extra math functions functions used by SiriDB.
+ */
+#ifndef XMATH_H_
+#define XMATH_H_
+
+#include <inttypes.h>
+#include <stddef.h>
+
+uint32_t xmath_ipow(int base, int exp);
+size_t xmath_max_size(size_t n, ...);
+
+#endif /* XMATH_H_ */
--- /dev/null
+/*
+ * xpath.h - Path and file tools.
+ */
+#ifndef XPATH_H_
+#define XPATH_H_
+
+#include <stdio.h>
+#include <limits.h>
+
+#ifndef PATH_MAX
+#define XPATH_MAX 4096
+#else
+#define XPATH_MAX PATH_MAX
+#endif
+
+int xpath_file_exist(const char * fn);
+int xpath_is_dir(const char * path);
+ssize_t xpath_get_content(char ** buffer, const char * fn);
+int xpath_get_exec_path(char * path);
+int xpath_rmdir(const char * path);
+
+#endif /* XPATH_H_ */
--- /dev/null
+/*
+ * xstr.h - Extra String functions used by SiriDB.
+ */
+#ifndef XSTR_H_
+#define XSTR_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <inttypes.h>
+
+void xstr_upper_case(char * sptr);
+void xstr_lower_case(char * sptr);
+void xstr_replace_char(char * sptr, char orig, char repl);
+int xstr_replace_str(char * str, char * o, char * r, size_t n);
+void xstr_split_join(char * pt, char split_chr, char join_chr);
+
+/* do not use trim when the char is created with *alloc */
+void xstr_trim(char ** str, char chr);
+bool xstr_is_empty(const char * str);
+bool xstr_is_int(const char * str);
+bool xstr_is_float(const char * str);
+bool xstr_is_graph(const char * str);
+double xstr_to_double(const char * src);
+uint64_t xstr_to_uint64(const char * src, size_t len);
+char * xstr_dup(const char * src, size_t * n);
+
+/* important: 'dest' needs to be freed */
+size_t xstr_extract_string(char * dest, const char * source, size_t len);
+
+#endif /* XSTR_H_ */
--- /dev/null
+testdir/
+Dockerfile
+*.log
+__pycache__/
--- /dev/null
+FROM ubuntu:18.04 as builder
+RUN apt-get update && \
+ apt-get install -y \
+ libcleri-dev \
+ libuv1-dev \
+ libpcre2-dev \
+ libyajl-dev \
+ uuid-dev \
+ build-essential
+COPY ./main.c ./main.c
+COPY ./src/ ./src/
+COPY ./include/ ./include/
+COPY ./Release/ ./Release/
+RUN cd ./Release && \
+ make clean && \
+ CFLAGS="-Werror -std=gnu89" make
+
+FROM python
+RUN apt-get update && \
+ apt-get install -y \
+ valgrind \
+ libuv1 \
+ libpcre2-8-0 \
+ libyajl2 && \
+ wget https://github.com/SiriDB/siridb-admin/releases/download/1.2.0/siridb-admin_1.2.0_linux_amd64.bin -O /usr/local/bin/siridb-admin && \
+ chmod +x /usr/local/bin/siridb-admin
+COPY --from=builder ./Release/siridb-server /Release/siridb-server
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libcleri* /usr/lib/x86_64-linux-gnu/
+COPY ./itest/ /itest/
+COPY ./help/ /help/
+COPY ./grammar/grammar.py /grammar/grammar.py
+COPY ./grammar/siridbhelp.py /grammar/siridbhelp.py
+WORKDIR /itest
+RUN pip install -r requirements.txt
+CMD [ "python", "run_all.py", "-m", "-b=Release" ]
--- /dev/null
+#!/usr/bin/python3
+import asyncio
+import aiohttp
+import argparse
+
+URL = \
+ 'http://www.google.com/finance/getprices?' \
+ 'i={interval}&p={days}d&f=d,o,h,l,c,v&df=cpct&q={ticker}'
+
+
+def _data_to_csv(data, ticker, interval):
+ lines = []
+ series = None
+ ts = None
+ tz_offset = None
+ for n, line in enumerate(data.splitlines()):
+ if line.startswith('COLUMNS='):
+ series = [
+ 'GOOGLE-FINANCE-{}-{}'.format(ticker, column)
+ for column in line[13:].split(',')]
+ lines.append(',' + ','.join(series))
+ elif line.startswith('TIMEZONE_OFFSET='):
+ tz_offset = int(line[17:])
+ elif tz_offset is not None:
+ date, *fields = line.split(',')
+ if date[0] == 'a':
+ ts = int(date[1:])
+ offset = 0
+ else:
+ offset = int(date) * interval
+ lines.append('{},{:.2f},{:.2f},{:.2f},{:.2f},{}'.format(
+ ts + offset, *map(float, fields[:-1]), fields[-1]))
+
+ if series is None:
+ raise Exception('Missing series in data')
+
+ return '\n'.join(lines)
+
+
+async def get_google_finance_data(ticker, interval, days):
+ """Returns Google Finance Data in CSV format.
+
+ :param ticker: This is the ticker symbol of the stock
+ :param interval: Interval or frequency in seconds (minimal 60)
+ :param days: The historical data period in days
+ :return: string (csv format)
+ """
+ assert interval >= 60, \
+ 'Inteval lower than 60 is not supported by the Google Finance API.'
+
+ async with aiohttp.ClientSession() as session:
+ async with session.get(URL.format(
+ interval=interval,
+ days=days,
+ ticker=ticker)) as resp:
+ content = await resp.text()
+ return _data_to_csv(content, ticker, interval)
+
+
+async def save_google_finance_data(fn, ticker, interval, days):
+ """Returns Google Finance Data in CSV format.
+
+ :param fn: Filename where the csv data is saved.
+ :param ticker: This is the ticker symbol of the stock
+ :param interval: Interval or frequency in seconds (minimal 60)
+ :param days: The historical data period in days
+ :return: None
+ """
+ csv_data = await get_google_finance_data(ticker, interval, days)
+ with open(fn, 'w') as f:
+ f.write(csv_data)
+
+
+async def print_google_finance_data(ticker, interval, days):
+ """Prints Google Finance Data in CSV format.
+
+ :param ticker: This is the ticker symbol of the stock
+ :param interval: Interval or frequency in seconds (minimal 60)
+ :param days: The historical data period in days
+ :return: None
+ """
+ print(await get_google_finance_data(ticker, interval, days))
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ '-t', '--ticker',
+ help='Ticker symbol of the stock',
+ default='IBM',
+ # required=True,
+ type=str)
+
+ parser.add_argument(
+ '-i', '--interval',
+ default=60,
+ help='Interval or frequency in seconds',
+ type=int)
+
+ parser.add_argument(
+ '-d', '--days',
+ default=10,
+ help='The historical data period in days',
+ type=int)
+
+ args = parser.parse_args()
+
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(print_google_finance_data(
+ ticker=args.ticker,
+ interval=args.interval,
+ days=args.days))
--- /dev/null
+k_map = {
+ 'r_doubleq_str': {
+ 'k_as': '"MERGED"',
+ 'k_suffix': '"SUFFIX"',
+ 'k_prefix': '"PREFIX"',
+ 'series_name': '"000000"',
+ 'k_filter': 10, # '"10"',
+ 'uuid': '"koos-VirtualBox:9010"',
+ 'k_name': '"000000"',
+ 'k_user': '"USER"',
+ 'k_password': '"PASSWORD"',
+ 'k_status': '"running"',
+ 'k_expression': '"/.*/"',
+ 'k_address': '"localhost"',
+ 'k_buffer_path': '"BUFFER_PATH"',
+ 'k_dbpath': '"DBPATH"',
+ 'k_uuid': '"UUID"',
+ 'k_version': '"VERSION"',
+ 'k_reindex_progress': '"REINDEX_PROGRESS"',
+ 'k_sync_progress': '"SYNC_PROGRESS"',
+ 'k_timezone': '"NAIVE"',
+ 'k_ip_support': '"ALL"',
+ 'k_libuv': '"1.8.0"',
+ 'k_server': '"SERVER"',
+ 'k_tee_pipe_name': '"PIPENAME"',
+ 'k_shard_duration': 86400,
+ 'k_expiration_num': 0,
+ 'k_expiration_log': 0,
+
+ 'aggregate_functions': '"1970-1-1 1:00:10"',
+ 'k_start': '"1970-1-1 1:00:00"',
+ 'k_after': '"1970-1-1 1:00:00"',
+ 'k_between': '"1970-1-1 1:00:00"',
+ 'k_before': '"1970-1-1 1:01:00"',
+ 'k_and': '"1970-1-1 1:01:00"',
+ 'k_end': '"1970-1-1 1:01:00"',
+ },
+ 'r_integer': {
+ 'k_series': 0, # GROUPS
+ 'k_active_handles': 0, # SERVERS
+ 'k_buffer_size': 0, # SERVERS
+ 'k_port': 9000, # SERVERS
+ 'k_startup_time': 0, # SERVERS
+ 'k_max_open_files': 0, # SERVERS
+ 'k_mem_usage': 0, # SERVERS
+ 'k_open_files': 0, # SERVERS
+ 'k_received_points': 0, # SERVERS
+ 'k_uptime': 0, # SERVERS,
+ 'k_servers': 0, # POOLS
+ 'k_limit': 10,
+ 'k_sid': 0,
+ 'k_pool': 0,
+ 'k_filter': 10,
+ 'k_size': 10,
+ 'k_length': 10,
+ 'aggregate_functions': 10,
+ 'k_start': 0,
+ 'k_after': 0,
+ 'k_between': 0,
+ 'k_before': 60,
+ 'k_and': 60,
+ 'k_end': 60,
+ 'k_shard_duration': 86400,
+ 'k_expiration_num': 0,
+ 'k_expiration_log': 0,
+ },
+ 'r_float': {
+ 'k_filter': 10.0,
+ 'k_drop_threshold': 0.99},
+ 'r_time_str': {
+ 'aggregate_functions': '10s',
+ 'k_start': '0d',
+ 'k_after': '0d',
+ 'k_between': '0d',
+ 'k_before': '1m',
+ 'k_and': '1m',
+ 'k_end': '1m',
+ 'k_shard_duration': '1d',
+ 'k_expiration_num': '0d',
+ 'k_expiration_log': '0d',
+ },
+ 'r_uuid_str': {
+ 'r_uuid_str': '"UUID"'},
+ 'r_uinteger': {
+ },
+ 'r_grave_str': {
+ 'group_name': '`GROUP`',
+ 'tag_name': '`TAG`',
+ 'group_tag_match': '`GROUP_OR_TAG`',
+ },
+ 'r_regex': {
+ 'r_regex': '/.*/'
+ },
+ 'r_comment': {
+ 'r_comment': '#',
+ },
+}
\ No newline at end of file
--- /dev/null
+import os
+import sys
+
+
+class QueryGenerator(list):
+
+ def __init__(self, grammar, maps={}):
+ self.grammar = grammar
+ self.ignores = maps.get('replace_map', {})
+ self.max_repeat_n_map = maps.get('max_repeat_n_map', {})
+ self.max_list_n_map = maps.get('max_list_n_map', {})
+ self.default_list_n = maps.get('default_list_n', 1)
+ self.regex_map = maps.get('regex_map', {})
+
+ def generate_queries(self, ename='START'):
+ try:
+ ele = getattr(self.grammar, ename)
+ except AttributeError:
+ print('{} element not found in tree'.format(ename))
+ else:
+ for q in self._addq([ele]):
+ yield ' '.join(map(str, q)).strip()
+
+ def _addq(self, q):
+ for i, e in enumerate(q):
+ if isinstance(e, str) or isinstance(e, int) or \
+ isinstance(e, float):
+ continue
+
+ ename = getattr(e, 'name', None)
+ iv = self.ignores.get(ename)
+ if iv is not None:
+ q[i] = iv
+ break
+
+ self.append(ename)
+ func = getattr(self, '_on_'+e.__class__.__name__, lambda q, i: [])
+ for q1 in func(q, i):
+ for q2 in self._addq(q1):
+ yield q2
+ self.pop()
+ q[i] = e
+ break
+ else:
+ yield q
+
+ def _on_Keyword(self, q, i):
+ q[i] = q[i]._keyword
+ yield q
+
+ def _on_Regex(self, q, i):
+ regex_ename = self[-1]
+ if regex_ename in self.regex_map:
+ re_map = self.regex_map[regex_ename]
+ for ename in self[::-1]:
+ val = re_map.get(ename)
+ if val is not None:
+ q[i] = val
+ yield q
+ break
+
+ # debug code
+
+ # else:
+ # print('no value found for:')
+ # print(ename)
+ # print(q[:i+1])
+ # print(i, len(self), self)
+ # print(self.regex_map)
+ # else:
+ # print('unknown regex element')
+ # print(ename)
+
+
+ def _on_Token(self, q, i):
+ q[i] = q[i]._token
+ yield q
+
+ def _on_Tokens(self, q, i):
+ for e1 in q[i]._tokens:
+ q[i] = e1
+ yield q
+
+ def _on_Sequence(self, q, i):
+ q = q[:i]+q[i]._elements+q[i+1:]
+ yield q
+
+ def _on_List(self, q, i):
+ if q[i]._min == 0:
+ q0 = q[:i]+q[i+1:]
+ yield q0
+
+ if q[i]._max == 1:
+ q0 = q[:i]+list(q[i]._elements)[:1]+q[i+1:]
+ yield q0
+ else:
+ name = getattr(q[i], 'name', getattr(q[i]._element, 'name', None))
+ n = self.max_list_n_map.get(name, self.default_list_n)
+ if q[i]._max:
+ n = min(n, q[i]._max)
+
+ eles = list(q[i]._elements)[:1] + \
+ [q[i]._delimiter, list(q[i]._elements)[0]]*(n-1)
+
+ q0 = q[:i]+eles+q[i+1:]
+ yield q0
+
+ def _on_Repeat(self, q, i):
+ if q[i]._min == 0:
+ q0 = q[:i]+q[i+1:]
+ yield q0
+
+ if q[i]._max == 1:
+ q0 = q[:i]+list(q[i]._elements)[:1]+q[i+1:]
+ yield q0
+ else:
+ n = self.max_repeat_n_map.get(getattr(q[i], 'name', None), 1)
+ eles = list(q[i]._elements)[:1]*n
+ q0 = q[:i]+eles+q[i+1:]
+ yield q0
+
+ def _on_Optional(self, q, i):
+ q[i] = list(q[i]._elements)[0]
+ yield q
+ q[i] = ''
+ yield q
+
+ def _on_Choice(self, q, i):
+ for e1 in q[i]._elements:
+ q[i] = e1
+ yield q
+
+ def _on_Rule(self, q, i):
+ for e1 in q[i]._element._elements:
+ if e1.__class__.__name__ == 'Sequence' and sum(
+ e2.__class__.__name__ == 'This' for e2 in e1._elements):
+ continue
+
+ q[i] = e1
+ yield q
--- /dev/null
+siridb-connector
+psutil
+requests
+pyleri
--- /dev/null
+#!/usr/bin/python3
+from testing import run_test
+from testing import Server
+from testing import parse_args
+from test_auto_duration import TestAutoDuration
+from test_buffer import TestBuffer
+from test_compression import TestCompression
+from test_create_database import TestCreateDatabase
+from test_expiration import TestExpiration
+from test_grammar import TestGrammar
+from test_group import TestGroup
+from test_http_api import TestHTTPAPI
+from test_insert import TestInsert
+from test_list import TestList
+from test_log import TestLog
+from test_parentheses import TestParenth
+from test_pipe_support import TestPipeSupport
+from test_pool import TestPool
+from test_select import TestSelect
+from test_select_ns import TestSelectNano
+from test_series import TestSeries
+from test_server import TestServer
+from test_tags import TestTags
+from test_tee import TestTee
+from test_user import TestUser
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestAutoDuration())
+ run_test(TestBuffer())
+ run_test(TestCompression())
+ run_test(TestCreateDatabase())
+ run_test(TestExpiration())
+ run_test(TestGrammar())
+ run_test(TestGroup())
+ run_test(TestHTTPAPI())
+ run_test(TestInsert())
+ run_test(TestList())
+ run_test(TestLog())
+ run_test(TestParenth())
+ run_test(TestPipeSupport())
+ run_test(TestPool())
+ run_test(TestSelect())
+ run_test(TestSelectNano())
+ run_test(TestSeries())
+ run_test(TestServer())
+ run_test(TestTags())
+ run_test(TestTee())
+ run_test(TestUser())
--- /dev/null
+#!/usr/bin/python3
+import os
+import sys
+import argparse
+import asyncio
+import time
+import logging
+import string
+import random
+import datetime
+import math
+import collections
+import signal
+from siridb.connector import SiriDBClient
+
+
+# Version information
+__version_info__ = (0, 0, 5)
+__version__ = '.'.join(map(str, __version_info__))
+__maintainer__ = 'Jeroen van der Heijden'
+__email__ = 'jeroen@transceptor.technology'
+
+
+# Logging definitions
+_LOG_DATE_FMT = '%y%m%d %H:%M:%S'
+_MAP_LOGLEVELS = {
+ 'DEBUG': logging.DEBUG,
+ 'INFO': logging.INFO,
+ 'WARNING': logging.WARNING,
+ 'ERROR': logging.ERROR,
+ 'CRITICAL': logging.CRITICAL
+}
+
+# counters etc.
+total_processed = 0
+total_failed = 0
+start_time = time.time()
+
+# Stop is used when pressing CTRL+C
+stop = False
+
+
+def setup_logger(args):
+ # setup formatter without using colors
+ formatter = logging.Formatter(
+ fmt='[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d] ' +
+ '%(message)s',
+ datefmt=_LOG_DATE_FMT,
+ style='%')
+
+ logger = logging.getLogger()
+
+ logger.setLevel(_MAP_LOGLEVELS[args.log_level.upper()])
+
+ if args.log_file_prefix:
+ # create file handler
+ ch = logging.RotatingFileHandler(
+ args.log_file_prefix,
+ maxBytes=args.log_file_max_size,
+ backupCount=args.log_file_num_backups)
+
+ else:
+ # create console handler
+ ch = logging.StreamHandler()
+
+ # we can set the handler level to DEBUG since we control the root level
+ ch.setLevel(logging.DEBUG)
+ ch.setFormatter(formatter)
+ logger.addHandler(ch)
+
+
+class Series:
+ # Class values are set with the .init() method.
+ _series = []
+ _timestamp = None # always in seconds
+ _ts_factor = None
+ _interval_range = None
+ _r = None
+
+ def __init__(self, r, allowed_kinds=(int, float, str), wrong_type=False):
+ self.kind = r.choice(allowed_kinds)
+ self.lval = {
+ str: '',
+ int: 0,
+ float: 0.0
+ }[self.kind]
+ self.lts = self._timestamp
+
+ factor = 10**r.randint(int(self.kind == int), 9)
+ self.random_range = (
+ int(r.random() * -factor),
+ int(r.random() * factor) + 1)
+ self.sign = 1
+
+ self.ignore_last = r.random() > 0.95
+ self.likely_equal = r.choice([0.01, 0.1, 0.2, 0.5, 0.99])
+ self.likely_change_sign = r.choice([0.0, 0.1, 0.25, 0.5, 0.9])
+
+ self.as_int = wrong_type and self.kind == float and r.random() > 0.9
+ self.likely_inf = r.random() * 0.2 \
+ if self.kind == float and r.random() > 0.95 else False
+ self.likely_nan = r.random() * 0.2 \
+ if self.kind == float and r.random() > 0.95 else False
+
+ self.gen_float = wrong_type and self.kind == int and r.random() > 0.97
+
+ self.name = self._gen_name()
+ Series._series.append(self)
+
+ def get_value(self):
+ if self._r.random() < self.likely_change_sign:
+ self.sign = -self.sign
+
+ if self._r.random() > self.likely_equal:
+ if self.kind == int:
+ val = self.sign * self._r.randint(*self.random_range)
+ if self.ignore_last:
+ self.lval = val
+ else:
+ self.lval += val
+ if self.lval.bit_length() > 63:
+ self.lval = 0
+
+ elif self.kind == float:
+ if self.likely_inf and self._r.random() < self.likely_inf:
+ return self.sign * math.inf
+ elif self.likely_nan and self._r.random() < self.likely_nan:
+ return math.nan
+ else:
+ val = self.sign * self._r.random() * self.random_range[1]
+ if self.ignore_last:
+ self.lval = val
+ else:
+ self.lval += val
+ if self.as_int:
+ self.lval = round(self.lval, 0)
+
+ if self.gen_float:
+ self.kind = float
+ self.gen_float = False
+
+ return self.lval
+
+ @classmethod
+ def init(cls, args, ts_factor):
+ cls._r = random.Random()
+ cls._r.seed(
+ time.time() if args.seed_data is None else args.seed_data)
+
+ series_rand = random.Random()
+ series_rand.seed(
+ time.time() if args.seed_series is None else args.seed_series)
+
+ cls._ts_factor = ts_factor
+ cls._timestamp = int(time.mktime(datetime.datetime.strptime(
+ args.start_date,
+ '%Y-%m-%d').timetuple()))
+
+ n = math.ceil(args.ts_interval * 0.05)
+ cls._interval_range = (-n, n)
+
+ translate = {
+ 'int': int,
+ 'float': float,
+ 'str': str}
+ kinds = [translate[k] for k in args.kinds]
+
+ for i in range(args.num_series):
+ Series(
+ r=series_rand,
+ allowed_kinds=kinds,
+ wrong_type=args.wrong_type)
+
+ def _gen_name(self):
+ name = '/n:{}/range:{},{}/eq:{}/cs:{}/opt:{}{}{}{}{}'.format(
+ len(self._series),
+ self.random_range[0],
+ self.random_range[1],
+ self.likely_equal,
+ self.likely_change_sign,
+ '(ign_last)' if self.ignore_last else '',
+ '(as_int)' if self.as_int else '',
+ '(gen_float)' if self.gen_float else '',
+ '(inf:{:.3f})'.format(self.likely_inf) if self.likely_inf else '',
+ '(nan:{:.3f})'.format(self.likely_nan) if self.likely_nan else '')
+
+ return name
+
+ def get_ts(self, now, args):
+ if args.ts_interval <= 0:
+ return self._r.randint(self._timestamp, now) * self._ts_factor
+ self.lts += args.ts_interval
+ if args.ts_randomize:
+ self.lts += self._r.randint(*self._interval_range)
+ return self.lts * self._ts_factor
+
+ @classmethod
+ def pick_series(cls, n):
+ series = cls._series[:]
+ cls._r.shuffle(series)
+ return series[:n]
+
+ @classmethod
+ def get_data(cls, args):
+ nseries, npoints = args.series_per_batch, args.points_per_batch
+
+ if nseries < 1:
+ nseries = cls._r.randint(1, min(10000, cls.num()))
+
+ if npoints < 1:
+ npoints = cls._r.randint(nseries, 10000)
+
+ series = cls.pick_series(nseries)
+ now = int(time.time())
+ data = collections.defaultdict(list)
+
+ for s in series:
+ val = s.get_value()
+ ts = s.get_ts(now, args)
+ data[s.name].append([ts, val])
+
+ npoints -= nseries
+ while npoints:
+ s = cls._r.choice(series)
+ ts = s.get_ts(now, args)
+ val = s.get_value()
+ data[s.name].append([ts, val])
+ npoints -= 1
+
+ return data
+
+ @classmethod
+ def num(cls):
+ return len(cls._series)
+
+
+async def get_ts_factor(siri):
+ res = await siri.query('show time_precision')
+ return 10**(['s', 'ms', 'us', 'ns'].index(res['data'][0]['value'])*3)
+
+
+def queue_data(args):
+ n = args.num_batches
+ while n and stop is False:
+ data = Series.get_data(args)
+ yield data
+ n -= 1
+
+
+async def siridb_insert(siri, data, task_counter):
+ '''Insert data into SiriDB.'''
+ global total_processed
+ global total_failed
+ global start_time
+
+ n = sum((len(p) for p in data.values()))
+ start = time.time()
+ try:
+ await siri.insert(data)
+ # await asyncio.sleep(0.1)
+ except Exception as e:
+ logging.exception(e)
+ total_failed += n
+ else:
+ total_processed += n
+ logging.info(
+ 'processed {} records, running {} task(s), {:d} records/sec'
+ .format(
+ total_processed,
+ len(task_counter) - 1,
+ int(total_processed // (time.time() - start_time))))
+ finally:
+ task_counter.pop()
+
+
+async def dump_data(siri, args):
+ task_counter = []
+
+ try:
+ await siri.connect()
+
+ ts_factor = await get_ts_factor(siri)
+ Series.init(args, ts_factor)
+
+ q = queue_data(args)
+ for data in q:
+
+ task_counter.append(1)
+ while len(task_counter) > args.max_parallel:
+ await asyncio.sleep(0.2)
+
+ asyncio.ensure_future(siridb_insert(siri, data, task_counter))
+
+ # sleep 0 so the async loop will run to pick-up tasks
+ await asyncio.sleep(args.sleep)
+
+ except Exception as e:
+ logging.exception(e)
+
+ finally:
+ while len(task_counter):
+ await asyncio.sleep(0.5)
+ siri.close()
+
+
+def signal_handler(signal, frame):
+ global stop
+ logging.warning(
+ 'Asked to stop, existing data in the queue will be finished...')
+ stop = True
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ '-u', '--user',
+ type=str,
+ default='iris',
+ help='database user')
+
+ parser.add_argument(
+ '-p', '--password',
+ type=str,
+ default='siri',
+ help='password')
+
+ parser.add_argument(
+ '-d', '--dbname',
+ type=str,
+ default='dbtest',
+ help='database name')
+
+ parser.add_argument(
+ '-s', '--servers',
+ type=str,
+ default='localhost:9000',
+ help='siridb server(s)')
+
+ parser.add_argument(
+ '--seed-series',
+ type=str,
+ help='Optional seed for generating series. '
+ 'If no seed is given, the current timestamp will be used.')
+
+ parser.add_argument(
+ '--seed-data',
+ type=str,
+ help='Optional seed for generating data. '
+ 'If no seed is given, the current timestamp will be used.')
+
+ parser.add_argument(
+ '-v', '--version',
+ action='store_true',
+ help='print version information and exit')
+
+ parser.add_argument(
+ '--start-date',
+ type=str,
+ default='2017-01-01',
+ help='Timestamps will be generated starting from this date. '
+ '(using the format YYYY-MM-DD')
+
+ parser.add_argument(
+ '--ts-interval',
+ type=int,
+ default=60,
+ help='Timestamp interval in seconds. '
+ 'When <= 0 a ramdom interval is used for each data point.')
+
+ parser.add_argument(
+ '--sleep',
+ type=float,
+ default=0.0,
+ help='Sleep between inserts in seconds. (default: 0.0 seconds)')
+
+ parser.add_argument(
+ '--ts-randomize',
+ action='store_true',
+ help='A random value will be added to the timestamp intervals so '
+ 'they will be not exact anymore. In case --ts-interval is '
+ '<= 0 this property will be ignored.')
+
+ parser.add_argument(
+ '--num-series',
+ type=int,
+ default=10000,
+ help='number of series to generate')
+
+ parser.add_argument(
+ '--num-batches',
+ type=int,
+ default=10,
+ help='number of batches to inserts. (value < 0 will be continues')
+
+ parser.add_argument(
+ '--series-per-batch',
+ type=int,
+ default=5000,
+ help='number of series per batch. (0 for random)')
+
+ parser.add_argument(
+ '--points-per-batch',
+ type=int,
+ default=10000,
+ help='number of points per batch. '
+ '(must be equal or larger than series-per-batch or 0 for random)')
+
+ parser.add_argument(
+ '--kinds',
+ nargs='+',
+ default=('int', 'float'),
+ choices=('int', 'float')) # , 'str'
+
+ parser.add_argument(
+ '--wrong-type',
+ action='store_true',
+ help='Allow series to insert points using a wrong type')
+
+ parser.add_argument(
+ '--max-parallel',
+ type=int,
+ default=3,
+ help='maximum number of allowed parallel inserts')
+
+ parser.add_argument(
+ '-l', '--log-level',
+ default='info',
+ help='set the log level (default: info)',
+ choices=['debug', 'info', 'warning', 'error'])
+
+ parser.add_argument(
+ '--log-file-max-size',
+ default=50000000,
+ help='max size of log files before rollover ' +
+ '(--log-file-prefix must be set)',
+ type=int)
+
+ parser.add_argument(
+ '--log-file-num-backups',
+ default=6,
+ help='number of log files to keep (--log-file-prefix must be set)',
+ type=int)
+
+ parser.add_argument(
+ '--log-file-prefix',
+ help='path prefix for log files (when not provided we send the ' +
+ 'output to the console)',
+ type=str)
+
+ args = parser.parse_args()
+
+ # respond to --version argument
+ if args.version:
+ sys.exit('''
+SiriDB Data Generator Script {version}
+Maintainer: {maintainer} <{email}>
+Home-page: https://github.com/transceptor-technology/siridb-email-check
+ '''.strip().format(version=__version__,
+ maintainer=__maintainer__,
+ email=__email__))
+
+ if args.num_series < args.series_per_batch:
+ exit('num-series must be equal or greater than series-per-batch')
+
+ if args.points_per_batch != 0 and \
+ args.points_per_batch < args.series_per_batch:
+ exit('points-per-batch must be equal or greater than series-per-batch')
+
+ if args.max_parallel < 1:
+ exit('max-parallel must be > 0')
+
+ try:
+ if datetime.datetime.strptime(
+ args.start_date, '%Y-%m-%d') >= datetime.datetime.today():
+ exit('start date must be a date before today.')
+ except Exception:
+ exit('invalid date: {}'.format(args.start_date))
+
+ setup_logger(args)
+ signal.signal(signal.SIGINT, signal_handler)
+
+ siri = SiriDBClient(
+ username=args.user,
+ password=args.password,
+ dbname=args.dbname,
+ hostlist=[
+ [s.strip() for s in server.split(':')]
+ for server in args.servers.split(',')])
+
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(dump_data(siri, args))
+
+ total_time = time.time() - start_time
+ logging.info(
+ 'total time: {:.3f} seconds, '
+ 'processed: {}, '
+ 'failed: {}'
+ .format(
+ total_time,
+ total_processed,
+ total_failed))
--- /dev/null
+#Refer: http://www.linuxfoundation.org/collaborate/workgroups/networking/netem#Delaying_only_some_traffic
+#Refer: http://www.bomisofmab.com/blog/?p=100
+#Refer: http://drija.com/linux/41983/simulating-a-low-bandwidth-high-latency-network-connection-on-linux/
+#Setup the rate control and delay
+sudo tc qdisc add dev lo root handle 1: htb default 12
+sudo tc class add dev lo parent 1:1 classid 1:12 htb rate 56kbps ceil 128kbps
+sudo tc qdisc add dev lo parent 1:12 netem delay 200ms
+
+#Remove the rate control/delay
+sudo tc qdisc del dev lo root
+
+#To see what is configured on an interface, do this
+sudo tc -s qdisc ls dev lo
+
+#Replace lo with eth0/wlan0 to limit speed from wide lan
\ No newline at end of file
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+import math
+import re
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+TIME_PRECISION = 's'
+factor = 1
+
+
+class TestAutoDuration(TestBase):
+ title = 'Test auto duration'
+
+ @default_test_setup(
+ 2,
+ compression=True,
+ buffer_size=1024,
+ time_precision=TIME_PRECISION,
+ auto_duration=True,
+ optimize_interval=20)
+ async def run(self):
+ await self.client0.connect()
+
+ await self.db.add_pool(self.server1, sleep=30)
+ await self.assertIsRunning(self.db, self.client0, timeout=30)
+
+ series = {}
+ end_td = int(time.time())
+ start_ts = end_td - (3600 * 24 * 7 * 10)
+ tests = [[300, 10], [60, 5], [3600, 30], [60, 90], [10, 1]]
+
+ for i, cfg in enumerate(tests):
+ interval, r = cfg
+ for nameval in [['int', 42], ['float', 3.14], ['str', 'hi']]:
+ name, val = nameval
+ series['{}-{}'.format(name, i)] = [
+ [(t + random.randint(-r, r)) * factor, val]
+ for t in range(start_ts, end_td, interval)
+ ]
+
+ self.assertEqual(
+ await self.client0.insert(series),
+ {'success_msg': 'Successfully inserted {} point(s).'.format(
+ 2484720)})
+
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestAutoDuration())
--- /dev/null
+import os
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+class TestBuffer(TestBase):
+ title = 'Test buffer object'
+
+ async def _add_points(self):
+ for series_name in ['iris', 'db', 'ligo', 'sasha']:
+ if series_name not in self.total:
+ self.total[series_name] = []
+ batches = sum([ord(c) for c in series_name]) % 100
+ for i in range(batches):
+ npoints = []
+ n = int(i**0.5 * 10000 % 5) + 1
+ for p in range(n):
+ self.ts += (n + 5000) if i % 2 else (n - 5000)
+ npoints.append([self.ts, i*1000+p])
+ self.total[series_name].extend(npoints)
+ self.total[series_name].sort()
+ await self.client0.insert({series_name: npoints})
+
+ async def _test_equal(self):
+ for series_name, points in self.total.items():
+ res = await self.client0.query(f'select * from "{series_name}"')
+ res = res[series_name]
+ self.assertEqual(len(points), len(res))
+ self.assertEqual(points, res)
+
+ async def _change_buf_size(self, buffer_size):
+ self.client0.close()
+ result = await self.server0.stop()
+ self.assertTrue(result)
+ self.server0.set_buffer_size(self.db, buffer_size)
+ await self.server0.start(sleep=20)
+ await self.client0.connect()
+ res = await self.client0.query('show buffer_size')
+ self.assertEqual(res['data'][0]['value'], buffer_size)
+ await self._test_equal()
+
+ async def _change_buf_path(self, buffer_path):
+ self.client0.close()
+ result = await self.server0.stop()
+ self.assertTrue(result)
+ self.server0.set_buffer_path(self.db, buffer_path)
+ await self.server0.start(sleep=20)
+ await self.client0.connect()
+ res = await self.client0.query('show buffer_path')
+ self.assertEqual(res['data'][0]['value'], buffer_path)
+ res = await self.client0.query('show open_files')
+ self.assertEqual(res['data'][0]['value'], 3)
+ res = await self.client0.query(
+ f'alter server {self.uuid} set backup_mode true')
+ await asyncio.sleep(5)
+ res = await self.client0.query('show open_files')
+ self.assertEqual(res['data'][0]['value'], 0)
+ res = await self.client0.query(
+ f'alter server {self.uuid} set backup_mode false')
+ await self._test_equal()
+
+ @default_test_setup(1, time_precision='s', compression=False)
+ async def run(self):
+ await self.client0.connect()
+
+ res = await self.client0.query('show uuid')
+ self.uuid = res['data'][0]['value']
+
+ self.ts = 1500000000
+ self.total = {}
+
+ await self._add_points()
+ await self._test_equal()
+
+ await self._change_buf_path(os.path.join(
+ self.server0.dbpath,
+ self.db.dbname,
+ '../buf/'))
+
+ await self._change_buf_size(8192)
+
+ await self._add_points()
+ await self._test_equal()
+
+ await self._change_buf_size(8192)
+ await self._change_buf_size(512)
+
+ await self._add_points()
+ await self._test_equal()
+
+ await self._change_buf_size(1024)
+
+ await self._change_buf_path(os.path.join(
+ self.server0.dbpath,
+ self.db.dbname,
+ 'buf/'))
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestBuffer())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+import math
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+DATA = {
+ 'series-001 float': [
+ [1471254705, 1.5],
+ [1471254707, -3.5],
+ [1471254710, -7.3]],
+ 'series-001 integer': [
+ [1471254705, 5],
+ [1471254708, -3],
+ [1471254710, -7]],
+ 'series-002 float': [
+ [1471254705, 3.5],
+ [1471254707, -2.5],
+ [1471254710, -8.3]],
+ 'series-002 integer': [
+ [1471254705, 4],
+ [1471254708, -1],
+ [1471254710, -8]],
+ 'aggr': [
+ [1447249033, 531], [1447249337, 534],
+ [1447249633, 535], [1447249937, 531],
+ [1447250249, 532], [1447250549, 537],
+ [1447250868, 530], [1447251168, 520],
+ [1447251449, 54], [1447251749, 54],
+ [1447252049, 513], [1447252349, 537],
+ [1447252649, 528], [1447252968, 531],
+ [1447253244, 533], [1447253549, 538],
+ [1447253849, 534], [1447254149, 532],
+ [1447254449, 533], [1447254748, 537]],
+ 'huge': [
+ [1471254705, 9223372036854775807],
+ [1471254706, 9223372036854775806],
+ [1471254707, 9223372036854775805],
+ [1471254708, 9223372036854775804]],
+ 'equal ts': [
+ [1471254705, 0], [1471254705, 1], [1471254705, 1],
+ [1471254707, 0], [1471254707, 1], [1471254708, 0],
+ ],
+ 'variance': [
+ [1471254705, 2.75], [1471254706, 1.75], [1471254707, 1.25],
+ [1471254708, 0.25], [1471254709, 0.5], [1471254710, 1.25],
+ [1471254711, 3.5]
+ ],
+ 'pvariance': [
+ [1471254705, 0.0], [1471254706, 0.25], [1471254707, 0.25],
+ [1471254708, 1.25], [1471254709, 1.5], [1471254710, 1.75],
+ [1471254711, 2.75], [1471254712, 3.25]
+ ],
+ 'filter': [
+ [1471254705, 5],
+ [1471254710, -3],
+ [1471254715, -7],
+ [1471254720, 7]
+ ],
+ 'one': [
+ [1471254710, 1]
+ ],
+ 'log': [
+ [1471254710, 'log line one'],
+ [1471254712, 'log line two'],
+ [1471254714, 'another line (three)'],
+ [1471254716, 'and yet one more'],
+ ],
+ 'special': [
+ [1471254705, 0.1],
+ [1471254706, math.nan],
+ [1471254707, math.inf],
+ [1471254708, -math.inf],
+ ]
+}
+
+
+class TestCluster(TestBase):
+ title = 'Test siridb-cluster'
+
+ @default_test_setup(3, time_precision='s')
+ async def run(self):
+ await self.client0.connect()
+
+ await self.client0.insert(DATA)
+
+ await self.client0.query('''
+ alter series /series.*/ tag `SERIES`
+ ''')
+
+ await asyncio.sleep(3.0)
+
+ await self.client0.query('''
+ alter series /.*/ - `SERIES` tag `OTHER`
+ ''')
+
+ await self.db.add_pool(self.server1)
+ await self.assertIsRunning(self.db, self.client0, timeout=30)
+
+ await asyncio.sleep(35)
+
+ await self.db.add_replica(self.server2, 0)
+ await self.assertIsRunning(self.db, self.client0, timeout=30)
+
+ # await asyncio.sleep(45)
+
+ # await self.db.add_replica(self.server3, 1)
+ # await self.assertIsRunning(self.db, self.client0, timeout=12)
+
+ # await asyncio.sleep(35)
+
+ # await self.db.add_pool(self.server4)
+ # await self.assertIsRunning(self.db, self.client0, timeout=12)
+
+ # await asyncio.sleep(35)
+
+ # await self.db.add_pool(self.server5)
+ # await self.assertIsRunning(self.db, self.client0, timeout=12)
+
+ # await self.db.add_replica(self.server1, 0)
+ # await asyncio.sleep(5)
+
+ # await self.db.add_replica(self.server3, 1)
+ # await asyncio.sleep(5)
+
+ # await self.db.add_replica(self.server5, 2)
+
+ # await self.assertIsRunning(self.db, self.client0, timeout=35)
+
+ # self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestCluster())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+TIME_PRECISION = 'ms'
+
+
+class TestCompression(TestBase):
+ title = 'Test compression'
+
+ GEN_POINTS = functools.partial(
+ gen_points, n=100, time_precision=TIME_PRECISION)
+
+ async def _test_series(self, client):
+
+ result = await client.query('select * from "series float"')
+ self.assertEqual(result['series float'], self.series_float)
+
+ result = await client.query('select * from "series int"')
+ self.assertEqual(result['series int'], self.series_int)
+
+ result = await client.query(
+ 'list series name, length, type, start, end')
+ result['series'].sort()
+ self.assertEqual(
+ result,
+ {'columns': ['name', 'length', 'type', 'start', 'end'],
+ 'series': [[
+ 'series float',
+ 10000, 'float',
+ self.series_float[0][0],
+ self.series_float[-1][0]], [
+ 'series int',
+ 10000, 'integer',
+ self.series_int[0][0],
+ self.series_int[-1][0]],
+ ]})
+
+ @default_test_setup(
+ 1,
+ time_precision=TIME_PRECISION,
+ optimize_interval=500,
+ compression=True)
+ async def run(self):
+ await self.client0.connect()
+
+ self.series_float = gen_points(
+ tp=float, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
+ random.shuffle(self.series_float)
+ self.series_int = gen_points(
+ tp=int, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
+ random.shuffle(self.series_int)
+
+ self.assertEqual(
+ await self.client0.insert({
+ 'series float': self.series_float,
+ 'series int': self.series_int
+ }), {'success_msg': 'Successfully inserted 20000 point(s).'})
+
+ self.series_float.sort()
+ self.series_int.sort()
+
+ await self._test_series(self.client0)
+
+ await self.client0.query('drop series /.*/ set ignore_threshold true')
+
+ # Create some random series and start 25 insert task parallel
+ series = gen_series(n=40)
+
+ for i in range(40):
+ await self.client0.insert_some_series(
+ series,
+ n=0.8,
+ timeout=0,
+ points=self.GEN_POINTS)
+
+ # Check the result
+ await self.assertSeries(self.client0, series)
+
+ for i in range(40):
+ await self.client0.insert_some_series(
+ series,
+ n=0.8,
+ timeout=0,
+ points=self.GEN_POINTS)
+
+ # Check the result
+ await self.assertSeries(self.client0, series)
+
+ self.client0.close()
+
+ result = await self.server0.stop()
+ self.assertTrue(result)
+
+ await self.server0.start(sleep=20)
+ await self.client0.connect()
+
+ # Check the result after rebooting the server
+ await self.assertSeries(self.client0, series)
+
+
+if __name__ == '__main__':
+ random.seed(1)
+ parse_args()
+ run_test(TestCompression())
--- /dev/null
+import asyncio
+import functools
+import os
+import subprocess
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+LENPOINTS = 12
+DATA = {
+ 'series-001': [
+ [1471254705000000005, 1.5],
+ [1471254705000000007, -3.5],
+ [1471254705000000010, -7.3]],
+ 'series-002': [
+ [1471254705000000005, 5],
+ [1471254705000000008, -3],
+ [1471254705000000010, -7]],
+ 'series-003': [
+ [1471254705000000005, 10.5],
+ [1471254705000000007, -8.5],
+ [1471254705000000010, -2.7]],
+ 'series-004': [
+ [1471254705000000005, 6],
+ [1471254705000000008, -8],
+ [1471254705000000010, -9]],
+}
+
+TIME_PRECISION = 'ns'
+
+
+class TestCreateDatabase(TestBase):
+ title = 'Test create database'
+
+ @default_test_setup(2, time_precision=TIME_PRECISION)
+ async def run(self):
+ await self.client0.connect()
+
+ tasks = [
+ asyncio.ensure_future(
+ SiriDB(
+ dbname="db_{}".format(i),
+ time_precision=TIME_PRECISION).create_on(
+ server=self.server0
+ ))
+ for i in range(30)]
+
+ await asyncio.gather(*tasks)
+
+ self.assertEqual(
+ await self.client0.insert(DATA),
+ {'success_msg': 'Successfully inserted {} point(s).'.format(
+ LENPOINTS)})
+
+ self.assertEqual(
+ await self.client0.query('create group `b` for /series.*/'),
+ {'success_msg': "Successfully created group 'b'."})
+
+ time.sleep(3)
+
+ self.client0.close()
+
+ for i in range(30):
+ client = Client(
+ db=SiriDB(dbname="db_{}".format(i)),
+ servers=self.servers)
+ await client.connect()
+ self.assertEqual(
+ await client.insert(DATA),
+ {'success_msg': 'Successfully inserted {} point(s).'.format(
+ LENPOINTS)})
+ client.close()
+
+ await self.client0.connect()
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select max(1) from /series.*/ '
+ 'merge as "max" using max(1)'),
+ {'max': [
+ [1471254705000000005, 10.5],
+ [1471254705000000007, -3.5],
+ [1471254705000000008, -3.0],
+ [1471254705000000010, -2.7]
+ ]})
+
+ await self.db.add_pool(self.server1)
+ await self.assertIsRunning(self.db, self.client0, timeout=20)
+
+ await asyncio.sleep(45)
+
+ await SiriDB(dbname="dbtest").drop_db(server=self.server1)
+
+ tasks = [
+ asyncio.ensure_future(
+ SiriDB(
+ dbname="db_{}".format(i)).drop_db(
+ server=self.server0
+ ))
+ for i in range(10)]
+
+ await asyncio.gather(*tasks)
+
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestCreateDatabase())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+TIME_PRECISION = 's'
+
+
+class TestExpiration(TestBase):
+ title = 'Test shard expiration'
+
+ GEN_POINTS = functools.partial(
+ gen_points, n=1, time_precision=TIME_PRECISION)
+
+ async def _test_series(self, client):
+
+ result = await client.query('select * from "series float"')
+ self.assertEqual(result['series float'], self.series_float)
+
+ result = await client.query('select * from "series int"')
+ self.assertEqual(result['series int'], self.series_int)
+
+ result = await client.query(
+ 'list series name, length, type, start, end')
+ result['series'].sort()
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name', 'length', 'type', 'start', 'end'],
+ 'series': [
+ [
+ 'series float',
+ 10000, 'float',
+ self.series_float[0][0],
+ self.series_float[-1][0]],
+ [
+ 'series int', 10000,
+ 'integer',
+ self.series_int[0][0],
+ self.series_int[-1][0]],
+ ]
+ })
+
+ async def insert(self, client, series, n, timeout=1):
+ for _ in range(n):
+ await client.insert_some_series(
+ series, timeout=timeout, points=self.GEN_POINTS)
+ await asyncio.sleep(1.0)
+
+ @default_test_setup(
+ 2,
+ time_precision=TIME_PRECISION,
+ compression=True,
+ optimize_interval=20)
+ async def run(self):
+ await self.client0.connect()
+
+ await self.db.add_replica(self.server1, 0, sleep=30)
+ # await self.db.add_pool(self.server1, sleep=30)
+
+ await self.assertIsRunning(self.db, self.client0, timeout=30)
+
+ await self.client1.connect()
+
+ self.series_float = gen_points(
+ tp=float, n=10000, time_precision=TIME_PRECISION, ts_gap='10m')
+ random.shuffle(self.series_float)
+
+ self.series_int = gen_points(
+ tp=int, n=10000, time_precision=TIME_PRECISION, ts_gap='10m')
+ random.shuffle(self.series_int)
+
+ self.assertEqual(
+ await self.client0.insert({
+ 'series float': self.series_float,
+ 'series int': self.series_int
+ }), {'success_msg': 'Successfully inserted 20000 point(s).'})
+
+ self.series_float.sort()
+ self.series_int.sort()
+
+ await self._test_series(self.client0)
+
+ total = (await self.client0.query('count shards'))['shards']
+ rest = (
+ await self.client0.query('count shards where end > now - 3w')
+ )['shards']
+
+ self.assertGreater(total, rest)
+
+ await self.client0.query('alter database set expiration_num 3w')
+ await asyncio.sleep(50) # wait for optimize to complete
+
+ total = (await self.client0.query('count shards'))['shards']
+ self.assertEqual(total, rest)
+
+ await self.client0.query('alter database set expiration_log 2w')
+ await self.client0.insert({
+ 'series_log': [
+ [int(time.time()) - 3600*24*15, "expired_log"]
+ ]
+ })
+
+ res = await self.client0.query('list series name, length "series_log"')
+ self.assertEqual(len(res['series']), 0)
+
+ await self.client0.insert({
+ 'series_log': [
+ [int(time.time()) - 3600*24*15, "expired_log"],
+ [int(time.time()) - 3600*24*7, "valid_log"],
+ ]
+ })
+
+ res = await self.client0.query('list series name, length "series_log"')
+ self.assertEqual(len(res['series']), 1)
+ self.assertEqual(res['series'], [['series_log', 1]])
+
+ await self.client0.query('alter database set drop_threshold 0.1')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "This query would drop .*"):
+ result = await self.client0.query(
+ 'alter database set expiration_num 1w')
+
+ total = (await self.client0.query('count shards'))['shards']
+ rest = (
+ await self.client0.query('count shards where end > now - 1w')
+ )['shards']
+
+ result = await self.client0.query(
+ 'alter database set expiration_num 1w '
+ 'set ignore_threshold true')
+
+ await asyncio.sleep(40) # wait for optimize to complete
+
+ total = (await self.client0.query('count shards'))['shards']
+ self.assertEqual(total, rest)
+
+ self.client0.close()
+ self.client1.close()
+
+
+if __name__ == '__main__':
+ random.seed(1)
+ parse_args()
+ run_test(TestExpiration())
--- /dev/null
+import re
+import random
+import asyncio
+import functools
+import random
+import time
+import os
+import sys
+from math import nan
+from testing import default_test_setup
+from testing import parse_args
+from testing import run_test
+from testing import Server
+from testing import SiriDB
+from testing import TestBase
+from testing.constants import PYGRAMMAR_PATH
+from querygenerator.querygenerator import QueryGenerator
+from querygenerator.k_map import k_map
+sys.path.append(PYGRAMMAR_PATH)
+from grammar import SiriGrammar # nopep8
+
+
+def gen_simple_data(m, n):
+ series = {
+ str(a).zfill(6): [[b*n+a, random.randint(0, 20)] for b in range(n)]
+ for a in range(m)}
+ return series
+
+
+def update_k_map_show(show):
+ kv = {a['name']: a['value'] for a in show['data']}
+ k_map['r_integer']['k_active_handles'] = kv['active_handles']
+ k_map['r_doubleq_str']['k_buffer_path'] = '"'+kv['buffer_path']+'"'
+ k_map['r_integer']['k_buffer_size'] = kv['buffer_size']
+ k_map['r_doubleq_str']['k_dbpath'] = '"'+kv['dbpath']+'"'
+ k_map['r_float']['k_drop_threshold'] = kv['drop_threshold']
+ k_map['r_doubleq_str']['k_ip_support'] = '"'+kv['ip_support']+'"'
+ k_map['r_doubleq_str']['k_libuv'] = '"'+kv['libuv']+'"'
+ k_map['r_integer']['k_max_open_files'] = kv['max_open_files']
+ k_map['r_integer']['k_mem_usage'] = kv['mem_usage']
+ k_map['r_integer']['k_open_files'] = kv['open_files']
+ k_map['r_integer']['k_pool'] = kv['pool']
+ k_map['r_integer']['k_received_points'] = kv['received_points']
+ k_map['r_uinteger']['k_list_limit'] = kv['list_limit']
+ k_map['r_integer']['k_startup_time'] = kv['startup_time']
+ k_map['r_doubleq_str']['k_status'] = '"'+kv['status']+'"'
+ k_map['r_doubleq_str']['k_sync_progress'] = '"'+kv['sync_progress']+'"'
+ k_map['r_doubleq_str']['k_timezone'] = '"'+kv['timezone']+'"'
+ k_map['r_integer']['k_uptime'] = kv['uptime']
+ k_map['r_uuid_str']['r_uuid_str'] = kv['uuid']
+ k_map['r_doubleq_str']['k_server'] = '"'+kv['server']+'"'
+ k_map['r_doubleq_str']['uuid'] = '"'+kv['server']+'"'
+ k_map['r_doubleq_str']['k_version'] = '"'+kv['version']+'"'
+ k_map['r_uinteger']['k_port'] = kv['server'].split(':', 1)[1]
+ k_map['r_uinteger']['k_select_points_limit'] = \
+ kv['select_points_limit']
+ k_map['r_doubleq_str']['k_reindex_progress'] = \
+ '"'+kv['reindex_progress']+'"'
+
+
+class TestGrammar(TestBase):
+ title = 'Test from grammar'
+
+ async def test_create_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': ''
+ }})
+ for q in qb.generate_queries('create_stmt'):
+ await self.client0.query(q)
+
+ async def test_select_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': '',
+ 'k_filter': '', # skip because only number type series
+ 'k_prefix': '', # skip
+ 'k_suffix': '', # skip
+ 'k_merge': '', # skip
+ 'k_where': '', # skip
+ 'after_expr': '', # skip
+ 'before_expr': '', # skip
+ 'between_expr': '', # skip
+ }
+ })
+ for q in qb.generate_queries('select_stmt'):
+ await self.client0.query(q)
+
+ async def test_revoke_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': ''
+ }})
+ for q in qb.generate_queries('revoke_stmt'):
+ await self.client0.query(q)
+
+ async def test_grant_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': ''
+ }})
+ for q in qb.generate_queries('grant_stmt'):
+ await self.client0.query(q)
+
+ async def test_alter_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': '',
+ 'k_now': '', # not possible (set expiration num/log)
+ 'set_name': '', # skip
+ 'set_address': '', # not possible
+ 'set_port': '', # not possible
+ 'set_timezone': '', # same value error
+ 'set_log_level': '', # not required, but skip to keep log level
+ }})
+ for q in qb.generate_queries('alter_stmt'):
+ await self.client0.query(q)
+
+ async def test_count_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': ''
+ }})
+ for q in qb.generate_queries('count_stmt'):
+ await self.client0.query(q)
+
+ async def test_list_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': ''
+ }})
+ for q in qb.generate_queries('list_stmt'):
+ await self.client0.query(q)
+
+ async def test_drop_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': '',
+ 'drop_server': '', # not possible
+ }})
+ for q in qb.generate_queries('drop_stmt'):
+ await self.client0.query(q)
+
+ async def test_show_stmt(self):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': ''
+ }})
+ for q in qb.generate_queries('show_stmt'):
+ await self.client0.query(q)
+
+ @default_test_setup(1)
+ async def run(self):
+ await self.client0.connect()
+
+ # await self.db.add_pool(self.server1, sleep=2)
+
+ update_k_map_show(await self.client0.query('show'))
+
+ series = gen_simple_data(20, 70)
+
+ await self.client0.insert(series)
+ await self.client0.query('create group `GROUP_OR_TAG` for /00000.*/')
+
+ await self.test_create_stmt()
+
+ time.sleep(2)
+
+ await self.test_count_stmt()
+
+ await self.test_list_stmt()
+
+ await self.test_select_stmt()
+
+ await self.test_revoke_stmt()
+
+ await self.test_grant_stmt()
+
+ await self.test_alter_stmt()
+
+ await self.test_drop_stmt()
+
+ await self.test_show_stmt()
+
+ self.client0.close()
+
+ return False
+
+class TestGrammarStart(TestBase):
+
+ async def test_all_stmts(self, client):
+ qb = QueryGenerator(SiriGrammar, {
+ 'regex_map': k_map,
+ 'replace_map': {
+ 'r_singleq_str': '',
+ 'r_comment': '',
+ 'k_timeit': '',
+ 'select_stmt': '',
+ 'list_stmt': '',
+ 'count_stmt': '',
+
+ 'alter_group': '',
+ #'drop_group': '',
+ 'alter_server': '',
+ 'drop_server': '',
+ 'alter_user': '',
+ 'drop_user': '',
+
+ #'set_address': '',
+ #'set_port': '',
+ 'set_timezone': '',
+ 'set_log_level': '', # not required, skip to keep log level
+ 'set_expiration_num': '',
+ 'set_expiration_log': '',
+
+ 'k_prefix': '',
+ 'k_suffix': '',
+ 'k_filter': '',
+ #'k_where': '',
+ 'after_expr': '',
+ 'before_expr': '',
+ 'between_expr': '',
+ 'k_merge': '',
+ }})
+ for q in qb.generate_queries():
+ await self.client0.query(q)
+
+ @default_test_setup(1)
+ async def run(self):
+ await self.client0.connect()
+
+ # await self.db.add_pool(self.server1, sleep=2)
+
+ update_k_map_show(await self.client0.query('show'))
+
+ series = gen_simple_data(20, 70)
+
+ await self.client0.insert(series)
+ await self.client0.query('create group `GROUP_OR_TAG` for /00000.*/')
+ #time.sleep(2)
+ await self.test_all_stmts()
+ self.client0.close()
+ return False
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestGrammar())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+DATA = {
+ 'a1': [[0, 0]],
+ 'a2': [[0, 0]],
+ 'b1': [[0, 0]],
+ 'b2': [[0, 0]],
+ 'c1': [[0, 0]],
+ 'c2': [[0, 0]]
+}
+
+
+class TestGroup(TestBase):
+ title = 'Test group object'
+
+ @default_test_setup(2)
+ async def run(self):
+ await self.client0.connect()
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Group name should be at least 1 characters.'):
+ await self.client0.query('create group `` for /c.*/')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Group name should be at least 1 characters.'):
+ await self.client0.query('create group `` for /c.*/')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Group name should be at most [0-9]+ characters.'):
+ await self.client0.query(
+ 'create group `{}` for /c.*/'.format('a' * 300))
+
+ self.assertEqual(
+ await self.client0.query('create group `a` for /a.*/'),
+ {'success_msg': "Successfully created group 'a'."})
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Group \'a\' already exists.'):
+ await self.client0.query('create group `a` for /a.*/')
+
+ self.assertEqual(
+ await self.client0.insert(DATA),
+ {'success_msg': 'Successfully inserted 6 point(s).'})
+
+ time.sleep(3)
+
+ self.assertEqual(
+ await self.client0.query('count series `a`'),
+ {'series': 2})
+
+ self.assertEqual(
+ await self.client0.query('create group `b` for /b.*/'),
+ {'success_msg': "Successfully created group 'b'."})
+
+ time.sleep(3)
+
+ self.assertEqual(
+ await self.client0.query('count series `b`'),
+ {'series': 2})
+
+ await self.db.add_pool(self.server1)
+ await self.assertIsRunning(self.db, self.client0, timeout=60)
+ await self.client1.connect()
+
+ self.assertEqual(
+ await self.client1.query('create group `c` for /c.*/'),
+ {'success_msg': "Successfully created group 'c'."})
+
+ time.sleep(3)
+
+ result = await self.client1.query('list groups series')
+ self.assertEqual(result.pop('groups'), [[2], [2], [2]])
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "Cannot compile regular expression.*"):
+ result = await self.client1.query('create group `invalid` for /(/')
+
+ self.assertEqual(
+ await self.client1.query('create group `one` for /.1/'),
+ {'success_msg': "Successfully created group 'one'."})
+
+ self.assertEqual(
+ await self.client1.query('alter group `one` set name "two"'),
+ {'success_msg': "Successfully updated group 'two'."})
+
+ self.assertEqual(
+ await self.client1.query('alter group `two` set expression /.2/'),
+ {'success_msg': "Successfully updated group 'two'."})
+
+ time.sleep(3)
+
+ result = await self.client0.query('list series `a` & `two`')
+ self.assertEqual(sorted(result.pop('series')), [['a2']])
+
+ result = await self.client0.query('list series `a` | `two`')
+ self.assertEqual(
+ sorted(result.pop('series')),
+ [['a1'], ['a2'], ['b2'], ['c2']])
+
+ result = await self.client0.query('list series `a` ^ `two`')
+ self.assertEqual(
+ sorted(result.pop('series')),
+ [['a1'], ['b2'], ['c2']])
+
+ result = await self.client0.query('list series `a` - `two`')
+ self.assertEqual(sorted(result.pop('series')), [['a1']])
+
+ result = await self.client0.query('list series `a`, `two` - "c2"')
+ self.assertEqual(
+ sorted(result.pop('series')),
+ [['a1'], ['a2'], ['b2']])
+
+ result = await self.client0.query('list series `a`, `two` & "c2"')
+ self.assertEqual(sorted(result.pop('series')), [['c2']])
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "Cannot compile regular expression.*"):
+ await self.client1.query('alter group `a` set expression /(.*/')
+
+ self.assertEqual(
+ await self.client1.query('alter group `a` set expression /b.*/'),
+ {'success_msg': "Successfully updated group 'a'."})
+
+ # await self.client0.query('drop group `a`')
+ # await self.client0.query('drop group `b`')
+ await self.client0.query('drop group `c`')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Group \'c\' does not exist.'):
+ await self.client0.query('drop group `c`')
+
+ await self.client0.query('create group `all` for /.*/ # bla')
+
+ await self.client0.query('alter group `all` set expression /.*/ # bla')
+
+ self.assertEqual(
+ await self.client0.query('count groups'),
+ {'groups': 4})
+
+ await asyncio.sleep(2)
+
+ self.assertEqual(
+ await self.client0.query('count groups where series > 2'),
+ {'groups': 2})
+
+ self.client0.close()
+ self.client1.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestGroup())
--- /dev/null
+import requests
+import json
+from testing import gen_points
+import asyncio
+import functools
+import random
+import time
+import math
+import re
+import qpack
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+TIME_PRECISION = 's'
+
+
+class TestHTTPAPI(TestBase):
+ title = 'Test HTTP API requests'
+
+ @default_test_setup(3, time_precision=TIME_PRECISION)
+ async def run(self):
+ await self.client0.connect()
+
+ x = requests.get(
+ f'http://localhost:9020/get-version', auth=('sa', 'siri'))
+
+ self.assertEqual(x.status_code, 200)
+ v = x.json()
+ self.assertTrue(isinstance(v, list))
+ self.assertTrue(isinstance(v[0], str))
+
+ x = requests.post(
+ f'http://localhost:9020/insert/dbtest',
+ auth=('iris', 'siri'),
+ headers={'Content-Type': 'application/json'})
+
+ self.assertEqual(x.status_code, 400)
+
+ series_float = gen_points(
+ tp=float, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
+
+ series_int = gen_points(
+ tp=int, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
+
+ data = {
+ 'my_float': series_float,
+ 'my_int': series_int
+ }
+
+ x = requests.post(
+ f'http://localhost:9020/insert/dbtest',
+ data=json.dumps(data),
+ auth=('iris', 'siri'),
+ headers={'Content-Type': 'application/json'}
+ )
+
+ self.assertEqual(x.status_code, 200)
+ self.assertDictEqual(x.json(), {
+ 'success_msg': 'Successfully inserted 20000 point(s).'})
+
+ data = {
+ 'dbname': 'dbtest',
+ 'host': 'localhost',
+ 'port': 9000,
+ 'username': 'iris',
+ 'password': 'siri'
+ }
+
+ x = requests.post(
+ f'http://localhost:9021/new-pool',
+ data=json.dumps(data),
+ auth=('sa', 'siri'),
+ headers={'Content-Type': 'application/json'})
+
+ self.assertEqual(x.status_code, 200)
+ self.assertEqual(x.json(), 'OK')
+
+ self.db.servers.append(self.server1)
+ await self.assertIsRunning(self.db, self.client0, timeout=30)
+
+ data = {'data': [[1579521271, 10], [1579521573, 20]]}
+ x = requests.post(
+ f'http://localhost:9020/insert/dbtest',
+ json=data,
+ auth=('iris', 'siri'))
+
+ self.assertEqual(x.status_code, 200)
+ self.assertDictEqual(x.json(), {
+ 'success_msg': 'Successfully inserted 2 point(s).'})
+
+ x = requests.post(
+ f'http://localhost:9020/query/dbtest',
+ json={'q': 'select * from "data"'},
+ auth=('iris', 'siri'))
+
+ self.assertEqual(x.status_code, 200)
+ self.assertEqual(x.json(), data)
+
+ x = requests.post(
+ f'http://localhost:9020/query/dbtest',
+ json={'q': 'select * from "data"', 't': 'ms'},
+ auth=('iris', 'siri'))
+
+ data = {
+ 'data': [[p[0] * 1000, p[1]] for p in data['data']]
+ }
+
+ self.assertEqual(x.status_code, 200)
+ self.assertEqual(x.json(), data)
+
+ x = requests.post(
+ f'http://localhost:9020/query/dbtest',
+ data=qpack.packb({
+ 'q': 'select sum(1579600000) from "data"',
+ 't': 'ms'}),
+ headers={'Content-Type': 'application/qpack'},
+ auth=('iris', 'siri'))
+
+ self.assertEqual(x.status_code, 200)
+ self.assertEqual(
+ qpack.unpackb(x.content, decode='utf8'),
+ {'data': [[1579600000000, 30]]})
+
+ x = requests.post(
+ f'http://localhost:9021/new-account',
+ json={'account': 't', 'password': ''},
+ auth=('sa', 'siri'))
+
+ self.assertEqual(x.status_code, 400)
+ self.assertEqual(x.json(), {
+ 'error_msg':
+ 'service account name should have at least 2 characters'})
+
+ x = requests.post(
+ f'http://localhost:9021/new-account',
+ json={'account': 'tt', 'password': 'pass'},
+ auth=('sa', 'siri'))
+
+ self.assertEqual(x.status_code, 200)
+
+ data = {
+ 'dbname': 'dbtest',
+ 'host': 'localhost',
+ 'port': 1234,
+ 'pool': 0,
+ 'username': 'iris',
+ 'password': 'siri'
+ }
+
+ auth = ('tt', 'pass')
+ x = requests.post(
+ f'http://localhost:9021/new-replica', json=data, auth=auth)
+
+ self.assertEqual(x.status_code, 400)
+ self.assertEqual(x.json(), {
+ 'error_msg': "database name already exists: 'dbtest'"})
+
+ x = requests.post(
+ f'http://localhost:9022/new-replica', json=data, auth=auth)
+ self.assertEqual(x.status_code, 401)
+
+ auth = ('sa', 'siri')
+ x = requests.post(
+ f'http://localhost:9022/new-replica', json=data, auth=auth)
+
+ self.assertEqual(x.status_code, 400)
+ self.assertEqual(x.json(), {
+ 'error_msg':
+ "connecting to server 'localhost:1234' failed with error: "
+ "connection refused"})
+
+ data['port'] = 9000
+ x = requests.post(
+ f'http://localhost:9022/new-replica', json=data, auth=auth)
+ self.assertEqual(x.status_code, 200)
+ self.assertEqual(x.json(), 'OK')
+
+ self.db.servers.append(self.server2)
+ await self.assertIsRunning(self.db, self.client0, timeout=50)
+
+ x = requests.get(
+ f'http://localhost:9022/get-databases', auth=auth)
+ self.assertEqual(x.status_code, 200)
+ self.assertEqual(x.json(), ['dbtest'])
+
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestHTTPAPI())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+TIME_PRECISION = 'ns'
+
+
+class TestInsert(TestBase):
+ title = 'Test inserts and response'
+
+ GEN_POINTS = functools.partial(
+ gen_points, n=1, time_precision=TIME_PRECISION)
+
+ async def _test_series(self, client):
+
+ result = await client.query('select * from "series float"')
+ self.assertEqual(result['series float'], self.series_float)
+
+ result = await client.query('select * from "series int"')
+ self.assertEqual(result['series int'], self.series_int)
+
+ result = await client.query(
+ 'list series name, length, type, start, end')
+ result['series'].sort()
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name', 'length', 'type', 'start', 'end'],
+ 'series': [
+ [
+ 'series float',
+ 10000, 'float',
+ self.series_float[0][0],
+ self.series_float[-1][0]],
+ [
+ 'series int', 10000,
+ 'integer',
+ self.series_int[0][0],
+ self.series_int[-1][0]],
+ ]
+ })
+
+ async def insert(self, client, series, n, timeout=1):
+ for _ in range(n):
+ await client.insert_some_series(
+ series, timeout=timeout, points=self.GEN_POINTS)
+ await asyncio.sleep(1.0)
+
+ @default_test_setup(2, time_precision=TIME_PRECISION, compression=False)
+ async def run(self):
+ await self.client0.connect()
+
+ self.assertEqual(
+ await self.client0.insert({}),
+ {'success_msg': 'Successfully inserted 0 point(s).'})
+
+ self.assertEqual(
+ await self.client0.insert([]),
+ {'success_msg': 'Successfully inserted 0 point(s).'})
+
+ self.series_float = gen_points(
+ tp=float, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
+ random.shuffle(self.series_float)
+ self.series_int = gen_points(
+ tp=int, n=10000, time_precision=TIME_PRECISION, ts_gap='5m')
+ random.shuffle(self.series_int)
+
+ self.assertEqual(
+ await self.client0.insert({
+ 'series float': self.series_float,
+ 'series int': self.series_int
+ }), {'success_msg': 'Successfully inserted 20000 point(s).'})
+
+ self.series_float.sort()
+ self.series_int.sort()
+
+ await self._test_series(self.client0)
+
+ with self.assertRaises(InsertError):
+ await self.client0.insert('[]')
+
+ with self.assertRaises(InsertError):
+ await self.client0.insert('[]')
+
+ with self.assertRaises(InsertError):
+ await self.client0.insert([{}])
+
+ with self.assertRaises(InsertError):
+ await self.client0.insert({'None': [[1, None]]})
+
+ with self.assertRaises(InsertError):
+ await self.client0.insert({'no points': []})
+
+ with self.assertRaises(InsertError):
+ await self.client0.insert({'no points': [[]]})
+
+ self.assertEqual(
+ await self.client0.insert({
+ 'ts_zero': [[0, 1]]
+ }), {'success_msg': 'Successfully inserted 1 point(s).'})
+
+ await self.client0.query('drop series "ts_zero"')
+
+ with self.assertRaises(InsertError):
+ await self.client0.insert([{'name': 'no points', 'points': []}])
+
+ # timestamps should be interger values
+ with self.assertRaises(InsertError):
+ await self.client0.insert({'invalid ts': [[0.5, 6]]})
+
+ # timestamps should be interger values
+ with self.assertRaises(InsertError):
+ await self.client0.insert(
+ {'invalid ts': [[-1, 6]]})
+
+ # empty series name is not allowed
+ with self.assertRaises(InsertError):
+ await self.client0.insert({'': [[1, 0]]})
+
+ # empty series name is not allowed
+ with self.assertRaises(InsertError):
+ await self.client0.insert([{'name': '', 'points': [[1, 0]]}])
+
+ await self.db.add_replica(self.server1, 0, sleep=30)
+ # await self.db.add_pool(self.server1, sleep=30)
+
+ await self.assertIsRunning(self.db, self.client0, timeout=30)
+
+ await self.client1.connect()
+
+ await self._test_series(self.client1)
+
+ # Create some random series and start 25 insert task parallel
+ series = gen_series(n=10000)
+ tasks = [
+ asyncio.ensure_future(
+ self.client0.insert_some_series(
+ series,
+ timeout=0,
+ points=self.GEN_POINTS))
+ for i in range(25)]
+ await asyncio.gather(*tasks)
+
+ await asyncio.sleep(2)
+
+ # Check the result
+ await self.assertSeries(self.client0, series)
+ await self.assertSeries(self.client1, series)
+
+ tasks = [
+ asyncio.ensure_future(self.client0.query(
+ 'drop series /.*/ set ignore_threshold true'))
+ for i in range(5)]
+
+ await asyncio.gather(*tasks)
+
+ tasks = [
+ asyncio.ensure_future(self.client0.query(
+ 'drop shards set ignore_threshold true'))
+ for i in range(5)]
+
+ await asyncio.gather(*tasks)
+
+ await asyncio.sleep(2)
+
+ self.client0.close()
+ self.client1.close()
+
+
+if __name__ == '__main__':
+ random.seed(1)
+ parse_args()
+ run_test(TestInsert())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+TIME_PRECISION = 's'
+
+
+class TestList(TestBase):
+ title = 'Test list'
+
+ GEN_POINTS = functools.partial(
+ gen_points, n=1, time_precision=TIME_PRECISION)
+
+ @default_test_setup(1, time_precision=TIME_PRECISION)
+ async def run(self):
+ await self.client0.connect()
+
+ # Create some random series and start 25 insert task parallel
+ series = gen_series(n=10000)
+ tasks = [
+ asyncio.ensure_future(
+ self.client0.insert_some_series(
+ series,
+ timeout=0,
+ points=self.GEN_POINTS))
+ for i in range(25)]
+
+ await asyncio.gather(*tasks)
+
+ await self.client0.query('list series /.*/ - /.*/')
+ await self.client0.query('list series /.*/ | /.*/')
+ await self.client0.query('list series /.*/ & /.*/')
+ await self.client0.query('list series /.*/ ^ /.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/')
+ await self.client0.query('list series /.*/ | /a.*/')
+ await self.client0.query('list series /.*/ & /a.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/ | /b.*/')
+ await self.client0.query('list series /.*/ | /a.*/ | /b.*/')
+ await self.client0.query('list series /.*/ & /a.*/ | /b.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/ | /b.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/ | /.*/')
+ await self.client0.query('list series /.*/ | /a.*/ | /.*/')
+ await self.client0.query('list series /.*/ & /a.*/ | /.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/ | /.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/ - /b.*/')
+ await self.client0.query('list series /.*/ | /a.*/ - /b.*/')
+ await self.client0.query('list series /.*/ & /a.*/ - /b.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/ - /b.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/ - /.*/')
+ await self.client0.query('list series /.*/ | /a.*/ - /.*/')
+ await self.client0.query('list series /.*/ & /a.*/ - /.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/ - /.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/ & /b.*/')
+ await self.client0.query('list series /.*/ | /a.*/ & /b.*/')
+ await self.client0.query('list series /.*/ & /a.*/ & /b.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/ & /b.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/ & /.*/')
+ await self.client0.query('list series /.*/ | /a.*/ & /.*/')
+ await self.client0.query('list series /.*/ & /a.*/ & /.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/ & /.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/ ^ /b.*/')
+ await self.client0.query('list series /.*/ | /a.*/ ^ /b.*/')
+ await self.client0.query('list series /.*/ & /a.*/ ^ /b.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/ ^ /b.*/')
+
+ await self.client0.query('list series /.*/ - /a.*/ ^ /.*/')
+ await self.client0.query('list series /.*/ | /a.*/ ^ /.*/')
+ await self.client0.query('list series /.*/ & /a.*/ ^ /.*/')
+ await self.client0.query('list series /.*/ ^ /a.*/ ^ /.*/')
+
+ await self.client0.query('alter database set list_limit 5000')
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Limit must be a value between 1 and 5000 '
+ 'but received: 6000.*'):
+ await self.client0.query(
+ 'list series limit 6000')
+
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestList())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+DATA = {
+ 'log': [
+ [1471254710, 'log line one'],
+ [1471254712, 'log line two'],
+ [1471254714, 'log line three'],
+ [1471254715, 'log line four'],
+ [1471254716, 'log line five'],
+ [1471254718, 'another line (six)'],
+ [1471254720, 'and yet one more, this is log line seven']
+ ],
+ 'utf16': [
+ [1471254710, ''],
+ [1471254712, ''],
+ [1471254714, ' '],
+ [1471254715, ' '],
+ [1471254716, ' '],
+ [1471254718, 'ԉ']
+ ],
+ 'long_log': [
+ [1471254710, '''
+time series is a series of data points indexed (or listed or graphed) in time order. Most commonly, a time series is a sequence taken at successive equally spaced points in time. Thus it is a sequence of discrete-time data. Examples of time series are heights of ocean tides, counts of sunspots, and the daily closing value of the Dow Jones Industrial Average.
+Time series are very frequently plotted via line charts. Time series are used in statistics, signal processing, pattern recognition, econometrics, mathematical finance, weather forecasting, earthquake prediction, electroencephalography, control engineering, astronomy, communications engineering, and largely in any domain of applied science and engineering which involves temporal measurements.
+Time series analysis comprises methods for analyzing time series data in order to extract meaningful statistics and other characteristics of the data. Time series forecasting is the use of a model to predict future values based on previously observed values. While regression analysis is often employed in such a way as to test theories that the current values of one or more independent time series affect the current value of another time series, this type of analysis of time series is not called "time series analysis", which focuses on comparing values of a single time series or multiple dependent time series at different points in time.[1] Interrupted time series analysis is the analysis of interventions on a single time series.
+Time series data have a natural temporal ordering. This makes time series analysis distinct from cross-sectional studies, in which there is no natural ordering of the observations (e.g. explaining people's wages by reference to their respective education levels, where the individuals' data could be entered in any order). Time series analysis is also distinct from spatial data analysis where the observations typically relate to geographical locations (e.g. accounting for house prices by the location as well as the intrinsic characteristics of the houses). A stochastic model for a time series will generally reflect the fact that observations close together in time will be more closely related than observations further apart. In addition, time series models will often make use of the natural one-way ordering of time so that values for a given period will be expressed as deriving in some way from past values, rather than from future values (see time reversibility.)
+Time series analysis can be applied to real-valued, continuous data, discrete numeric data, or discrete symbolic data (i.e. sequences of characters, such as letters and words in the English language[2]).']
+'''], # nopep8
+ [1471254712, '''
+Methods for time series analysis may be divided into two classes: frequency-domain methods and time-domain methods. The former include spectral analysis and wavelet analysis; the latter include auto-correlation and cross-correlation analysis. In the time domain, correlation and analysis can be made in a filter-like manner using scaled correlation, thereby mitigating the need to operate in the frequency domain.
+Additionally, time series analysis techniques may be divided into parametric and non-parametric methods. The parametric approaches assume that the underlying stationary stochastic process has a certain structure which can be described using a small number of parameters (for example, using an autoregressive or moving average model). In these approaches, the task is to estimate the parameters of the model that describes the stochastic process. By contrast, non-parametric approaches explicitly estimate the covariance or the spectrum of the process without assuming that the process has any particular structure.
+Methods of time series analysis may also be divided into linear and non-linear, and univariate and multivariate.
+'''], # nopep8
+ [1471254714, '''
+Curve fitting
+Main article: Curve fitting
+Curve fitting[5][6] is the process of constructing a curve, or mathematical function, that has the best fit to a series of data points,[7] possibly subject to constraints.[8][9] Curve fitting can involve either interpolation,[10][11] where an exact fit to the data is required, or smoothing,[12][13] in which a "smooth" function is constructed that approximately fits the data. A related topic is regression analysis,[14][15] which focuses more on questions of statistical inference such as how much uncertainty is present in a curve that is fit to data observed with random errors. Fitted curves can be used as an aid for data visualization,[16][17] to infer values of a function where no data are available,[18] and to summarize the relationships among two or more variables.[19] Extrapolation refers to the use of a fitted curve beyond the range of the observed data,[20] and is subject to a degree of uncertainty[21] since it may reflect the method used to construct the curve as much as it reflects the observed data.
+The construction of economic time series involves the estimation of some components for some dates by interpolation between values ("benchmarks") for earlier and later dates. Interpolation is estimation of an unknown quantity between two known quantities (historical data), or drawing conclusions about missing information from the available information ("reading between the lines").[22] Interpolation is useful where the data surrounding the missing data is available and its trend, seasonality, and longer-term cycles are known. This is often done by using a related series known for all relevant dates.[23] Alternatively polynomial interpolation or spline interpolation is used where piecewise polynomial functions are fit into time intervals such that they fit smoothly together. A different problem which is closely related to interpolation is the approximation of a complicated function by a simple function (also called regression).The main difference between regression and interpolation is that polynomial regression gives a single polynomial that models the entire data set. Spline interpolation, however, yield a piecewise continuous function composed of many polynomials to model the data set.
+Extrapolation is the process of estimating, beyond the original observation range, the value of a variable on the basis of its relationship with another variable. It is similar to interpolation, which produces estimates between known observations, but extrapolation is subject to greater uncertainty and a higher risk of producing meaningless results.
+
+'''], # nopep8
+ [1471254715, '''
+Function approximation
+Main article: Function approximation
+In general, a function approximation problem asks us to select a function among a well-defined class that closely matches ("approximates") a target function in a task-specific way. One can distinguish two major classes of function approximation problems: First, for known target functions approximation theory is the branch of numerical analysis that investigates how certain known functions (for example, special functions) can be approximated by a specific class of functions (for example, polynomials or rational functions) that often have desirable properties (inexpensive computation, continuity, integral and limit values, etc.).
+Second, the target function, call it g, may be unknown; instead of an explicit formula, only a set of points (a time series) of the form (x, g(x)) is provided. Depending on the structure of the domain and codomain of g, several techniques for approximating g may be applicable. For example, if g is an operation on the real numbers, techniques of interpolation, extrapolation, regression analysis, and curve fitting can be used. If the codomain (range or target set) of g is a finite set, one is dealing with a classification problem instead. A related problem of online time series approximation[24] is to summarize the data in one-pass and construct an approximate representation that can support a variety of time series queries with bounds on worst-case error.
+To some extent the different problems (regression, classification, fitness approximation) have received a unified treatment in statistical learning theory, where they are viewed as supervised learning problems.
+'''], # nopep8
+ [1471254716, '''
+Prediction and forecasting
+In statistics, prediction is a part of statistical inference. One particular approach to such inference is known as predictive inference, but the prediction can be undertaken within any of the several approaches to statistical inference. Indeed, one description of statistics is that it provides a means of transferring knowledge about a sample of a population to the whole population, and to other related populations, which is not necessarily the same as prediction over time. When information is transferred across time, often to specific points in time, the process is known as forecasting.
+Fully formed statistical models for stochastic simulation purposes, so as to generate alternative versions of the time series, representing what might happen over non-specific time-periods in the future
+Simple or fully formed statistical models to describe the likely outcome of the time series in the immediate future, given knowledge of the most recent outcomes (forecasting).
+Forecasting on time series is usually done using automated statistical software packages and programming languages, such as R, S, SAS, SPSS, Minitab, pandas (Python) and many others.
+Forecasting on large scale data is done using Spark which has spark-ts as a third party package.
+Classification
+Main article: Statistical classification
+Assigning time series pattern to a specific category, for example identify a word based on series of hand movements in sign language.
+Signal estimation
+See also: Signal processing and Estimation theory
+This approach is based on harmonic analysis and filtering of signals in the frequency domain using the Fourier transform, and spectral density estimation, the development of which was significantly accelerated during World War II by mathematician Norbert Wiener, electrical engineers Rudolf E. Kálmán, Dennis Gabor and others for filtering signals from noise and predicting signal values at a certain point in time. See Kalman filter, Estimation theory, and Digital signal processing
+'''], # nopep8
+ [1471254718, '''
+Segmentation
+Main article: Time-series segmentation
+Splitting a time-series into a sequence of segments. It is often the case that a time-series can be represented as a sequence of individual segments, each with its own characteristic properties. For example, the audio signal from a conference call can be partitioned into pieces corresponding to the times during which each person was speaking. In time-series segmentation, the goal is to identify the segment boundary points in the time-series, and to characterize the dynamical properties associated with each segment. One can approach this problem using change-point detection, or by modeling the time-series as a more sophisticated system, such as a Markov jump linear system.
+Models
+Models for time series data can have many forms and represent different stochastic processes. When modeling variations in the level of a process, three broad classes of practical importance are the autoregressive (AR) models, the integrated (I) models, and the moving average (MA) models. These three classes depend linearly on previous data points.[25] Combinations of these ideas produce autoregressive moving average (ARMA) and autoregressive integrated moving average (ARIMA) models. The autoregressive fractionally integrated moving average (ARFIMA) model generalizes the former three. Extensions of these classes to deal with vector-valued data are available under the heading of multivariate time-series models and sometimes the preceding acronyms are extended by including an initial "V" for "vector", as in VAR for vector autoregression. An additional set of extensions of these models is available for use where the observed time-series is driven by some "forcing" time-series (which may not have a causal effect on the observed series): the distinction from the multivariate case is that the forcing series may be deterministic or under the experimenter's control. For these models, the acronyms are extended with a final "X" for "exogenous".
+Non-linear dependence of the level of a series on previous data points is of interest, partly because of the possibility of producing a chaotic time series. However, more importantly, empirical investigations can indicate the advantage of using predictions derived from non-linear models, over those from linear models, as for example in nonlinear autoregressive exogenous models. Further references on nonlinear time series analysis: (Kantz and Schreiber),[26] and (Abarbanel)[27]
+Among other types of non-linear time series models, there are models to represent the changes of variance over time (heteroskedasticity). These models represent autoregressive conditional heteroskedasticity (ARCH) and the collection comprises a wide variety of representation (GARCH, TARCH, EGARCH, FIGARCH, CGARCH, etc.). Here changes in variability are related to, or predicted by, recent past values of the observed series. This is in contrast to other possible representations of locally varying variability, where the variability might be modelled as being driven by a separate time-varying process, as in a doubly stochastic model.
+In recent work on model-free analyses, wavelet transform based methods (for example locally stationary wavelets and wavelet decomposed neural networks) have gained favor. Multiscale (often referred to as multiresolution) techniques decompose a given time series, attempting to illustrate time dependence at multiple scales. See also Markov switching multifractal (MSMF) techniques for modeling volatility evolution.
+A Hidden Markov model (HMM) is a statistical Markov model in which the system being modeled is assumed to be a Markov process with unobserved (hidden) states. An HMM can be considered as the simplest dynamic Bayesian network. HMM models are widely used in speech recognition, for translating a time series of spoken words into text.
+'''], # nopep8
+ [1471254720, '''
+Time
+The flow of sand in an hourglass can be used to measure the passage of time. It also concretely represents the present as being between the past and the future.
+Time is the indefinite continued progress of existence and events that occur in apparently irreversible succession from the past through the present to the future.[1][2][3] Time is a component quantity of various measurements used to sequence events, to compare the duration of events or the intervals between them, and to quantify rates of change of quantities in material reality or in the conscious experience.[4][5][6][7] Time is often referred to as a fourth dimension, along with three spatial dimensions.[8]
+Time has long been an important subject of study in religion, philosophy, and science, but defining it in a manner applicable to all fields without circularity has consistently eluded scholars.[2][6][7][9][10][11] Nevertheless, diverse fields such as business, industry, sports, the sciences, and the performing arts all incorporate some notion of time into their respective measuring systems.[12][13][14]
+Two contrasting viewpoints on time divide prominent philosophers. One view is that time is part of the fundamental structure of the universe – a dimension independent of events, in which events occur in sequence. Isaac Newton subscribed to this realist view, and hence it is sometimes referred to as Newtonian time.[15][16] The opposing view is that time does not refer to any kind of "container" that events and objects "move through", nor to any entity that "flows", but that it is instead part of a fundamental intellectual structure (together with space and number) within which humans sequence and compare events. This second view, in the tradition of Gottfried Leibniz[17] and Immanuel Kant,[18][19] holds that time is neither an event nor a thing, and thus is not itself measurable nor can it be travelled.
+Time in physics is unambiguously operationally defined as "what a clock reads".[6][17][20] See Units of Time. Time is one of the seven fundamental physical quantities in both the International System of Units and International System of Quantities. Time is used to define other quantities – such as velocity – so defining time in terms of such quantities would result in circularity of definition.[21] An operational definition of time, wherein one says that observing a certain number of repetitions of one or another standard cyclical event (such as the passage of a free-swinging pendulum) constitutes one standard unit such as the second, is highly useful in the conduct of both advanced experiments and everyday affairs of life. The operational definition leaves aside the question whether there is something called time, apart from the counting activity just mentioned, that flows and that can be measured. Investigations of a single continuum called spacetime bring questions about space into questions about time, questions that have their roots in the works of early students of natural philosophy.
+Temporal measurement has occupied scientists and technologists, and was a prime motivation in navigation and astronomy. Periodic events and periodic motion have long served as standards for units of time. Examples include the apparent motion of the sun across the sky, the phases of the moon, the swing of a pendulum, and the beat of a heart. Currently, the international unit of time, the second, is defined by measuring the electronic transition frequency of caesium atoms (see below). Time is also of significant social importance, having economic value ("time is money") as well as personal value, due to an awareness of the limited time in each day and in human life spans.]
+Temporal measurement and history
+Generally speaking, methods of temporal measurement, or chronometry, take two distinct forms: the calendar, a mathematical tool for organising intervals of time,[22] and the clock, a physical mechanism that counts the passage of time. In day-to-day life, the clock is consulted for periods less than a day whereas the calendar is consulted for periods longer than a day. Increasingly, personal electronic devices display both calendars and clocks simultaneously. The number (as on a clock dial or calendar) that marks the occurrence of a specified event as to hour or date is obtained by counting from a fiducial epoch – a central reference point.
+History of the calendar
+Main article: Calendar
+Artifacts from the Paleolithic suggest that the moon was used to reckon time as early as 6,000 years ago.[23] Lunar calendars were among the first to appear, either 12 or 13 lunar months (either 354 or 384 days). Without intercalation to add days or months to some years, seasons quickly drift in a calendar based solely on twelve lunar months. Lunisolar calendars have a thirteenth month added to some years to make up for the difference between a full year (now known to be about 365.24 days) and a year of just twelve lunar months. The numbers twelve and thirteen came to feature prominently in many cultures, at least partly due to this relationship of months to years. Other early forms of calendars originated in Mesoamerica, particularly in ancient Mayan civilization. These calendars were religiously and astronomically based, with 18 months in a year and 20 days in a month, plus five epagomenal days at the end of the year.[24]
+The reforms of Julius Caesar in 45 BC put the Roman world on a solar calendar. This Julian calendar was faulty in that its intercalation still allowed the astronomical solstices and equinoxes to advance against it by about 11 minutes per year. Pope Gregory XIII introduced a correction in 1582; the Gregorian calendar was only slowly adopted by different nations over a period of centuries, but it is now the most commonly used calendar around the world, by far.
+During the French Revolution, a new clock and calendar were invented in attempt to de-Christianize time and create a more rational system in order to replace the Gregorian calendar. The French Republican Calendar's days consisted of ten hours of a hundred minutes of a hundred seconds, which marked a deviation from the 12-based duodecimal system used in many other devices by many cultures. The system was later abolished in 1806.[25]
+History of time measurement devices
+Horizontal sundial in Taganrog
+An old kitchen clock
+Main article: History of timekeeping devices
+See also: Clock
+A large variety of devices have been invented to measure time. The study of these devices is called horology.
+An Egyptian device that dates to c.1500 BC, similar in shape to a bent T-square, measured the passage of time from the shadow cast by its crossbar on a nonlinear rule. The T was oriented eastward in the mornings. At noon, the device was turned around so that it could cast its shadow in the evening direction.[26]
+A sundial uses a gnomon to cast a shadow on a set of markings calibrated to the hour. The position of the shadow marks the hour in local time. The idea to separate the day into smaller parts is credited to Egyptians because of their sundials, which operated on a duodecimal system. The importance of the number 12 is due the number of lunar cycles in a year and the number of stars used to count the passage of night.[27]
+The most precise timekeeping device of the ancient world was the water clock, or clepsydra, one of which was found in the tomb of Egyptian pharaoh Amenhotep I (1525–1504 BC). They could be used to measure the hours even at night, but required manual upkeep to replenish the flow of water. The Ancient Greeks and the people from Chaldea (southeastern Mesopotamia) regularly maintained timekeeping records as an essential part of their astronomical observations. Arab inventors and engineers in particular made improvements on the use of water clocks up to the Middle Ages.[28] In the 11th century, Chinese inventors and engineers invented the first mechanical clocks driven by an escapement mechanism.
+The hourglass uses the flow of sand to measure the flow of time. They were used in navigation. Ferdinand Magellan used 18 glasses on each ship for his circumnavigation of the globe (1522).[29] Incense sticks and candles were, and are, commonly used to measure time in temples and churches across the globe. Waterclocks, and later, mechanical clocks, were used to mark the events of the abbeys and monasteries of the Middle Ages. Richard of Wallingford (1292–1336), abbot of St. Alban's abbey, famously built a mechanical clock as an astronomical orrery about 1330.[30][31] Great advances in accurate time-keeping were made by Galileo Galilei and especially Christiaan Huygens with the invention of pendulum driven clocks along with the invention of the minute hand by Jost Burgi.[32]
+The English word clock probably comes from the Middle Dutch word klocke which, in turn, derives from the medieval Latin word clocca, which ultimately derives from Celtic and is cognate with French, Latin, and German words that mean bell. The passage of the hours at sea were marked by bells, and denoted the time (see ship's bell). The hours were marked by bells in abbeys as well as at sea.
+Clocks can range from watches, to more exotic varieties such as the Clock of the Long Now. They can be driven by a variety of means, including gravity, springs, and various forms of electrical power, and regulated by a variety of means such as a pendulum.
+The English word clock probably comes from the Middle Dutch word klocke which, in turn, derives from the medieval Latin word clocca, which ultimately derives from Celtic and is cognate with French, Latin, and German words that mean bell. The passage of the hours at sea were marked by bells, and denoted the time (see ship's bell). The hours were marked by bells in abbeys as well as at sea.
+Chip-scale atomic clocks, such as this one unveiled in 2004, are expected to greatly improve GPS location.[33]
+Clocks can range from watches, to more exotic varieties such as the Clock of the Long Now. They can be driven by a variety of means, including gravity, springs, and various forms of electrical power, and regulated by a variety of means such as a pendulum.
+Alarm clocks first appeared in ancient Greece around 250 BC with a water clock that would set off a whistle. This idea was later mechanized by Levi Hutchins and Seth E. Thomas
+A chronometer is a portable timekeeper that meets certain precision standards. Initially, the term was used to refer to the marine chronometer, a timepiece used to determine longitude by means of celestial navigation, a precision firstly achieved by John Harrison. More recently, the term has also been applied to the chronometer watch, a watch that meets precision standards set by the Swiss agency COSC.
+The most accurate timekeeping devices are atomic clocks, which are accurate to seconds in many millions of years,[34] and are used to calibrate other clocks and timekeeping instruments. Atomic clocks use the frequency of electronic transitions in certain atoms to measure the second. One of the most common atoms used is caesium, most modern atomic clocks probe caesium with microwaves to determine the frequency of these electron vibrations.[35] Since 1967, the International System of Measurements bases its unit of time, the second, on the properties of caesium atoms. SI defines the second as 9,192,631,770 cycles of the radiation that corresponds to the transition between two electron spin energy levels of the ground state of the 133Cs atom.
+Today, the Global Positioning System in coordination with the Network Time Protocol can be used to synchronize timekeeping systems across the globe.
+Definitions and standards
+The Mean Solar Time system defines the second as 1/86,400 of the mean solar day, which is the year-average of the solar day. The solar day is the time interval between two successive solar noons, i.e., the time interval between two successive passages of the Sun across the local meridian. The local meridian is an imaginary line that runs from celestial north pole to celestial south pole passing directly over the head of the observer. At the local meridian the Sun reaches its highest point on its daily arc across the sky.
+In 1874 the British Association for the Advancement of Science introduced the CGS (centimetre/gramme/second system) combining fundamental units of length, mass and time. The second is "elastic", because tidal friction is slowing the earth's rotation rate. For use in calculating ephemerides of celestial motion, therefore, in 1952 astronomers introduced the "ephemeris second", currently defined as the fraction 1/31,556,925.9747 of the tropical year for 1900 January 0 at 12 hours ephemeris time.
+The CGS system has been superseded by the Système international. The SI base unit for time is the SI second. The International System of Quantities, which incorporates the SI, also defines larger units of time equal to fixed integer multiples of one second (1 s), such as the minute, hour and day. These are not part of the SI, but may be used alongside the SI. Other units of time such as the month and the year are not equal to fixed multiples of 1 s, and instead exhibit significant variations in duration.
+The official SI definition of the second is as follows: The second is the duration of 9,192,631,770 periods of the radiation corresponding to the transition between the two hyperfine levels of the ground state of the caesium 133 atom.
+At its 1997 meeting, the CIPM affirmed that this definition refers to a caesium atom in its ground state at a temperature of 0 K.
+The current definition of the second, coupled with the current definition of the metre, is based on the special theory of relativity, which affirms our spacetime to be a Minkowski space. The definition of the second in mean solar time, however, is unchanged.
+World time
+While in theory, the concept of a single worldwide universal time-scale may have been conceived of many centuries ago, in practicality the technical ability to create and maintain such a time-scale did not become possible until the mid-19th century. The timescale adopted was Greenwich Mean Time, created in 1847. A few countries have replaced it with Coordinated Universal Time, UTC.
+History of the development of UTC
+With the advent of the industrial revolution, a greater understanding and agreement on the nature of time itself became increasingly necessary and helpful. In 1847 in Britain, Greenwich Mean Time (GMT) was first created for use by the British railways, the British navy, and the British shipping industry. Using telescopes, GMT was calibrated to the mean solar time at the Royal Observatory, Greenwich in the UK.
+As international commerce continued to increase throughout Europe, in order to achieve a more efficiently functioning modern society, an agreed upon, and highly accurate international standard of time measurement became necessary. In order to find or determine such a time-standard, three steps had to be followed:
+An internationally agreed upon time-standard had to be defined.
+This new time-standard then had to be consistently and accurately measured.
+The new time-standard then had to be freely shared and distributed around the world.
+The development of what is now known as UTC time came about historically as an effort which first began as a collaboration between 41 nations, officially agreed to and signed at the International Meridian Conference, in Washington D.C. in 1884. At this conference, the local mean solar time at the Royal Observatory, Greenwich in England was chosen to define the "universal day", counted from 0 hours at Greenwich mean midnight. This agreed with the civil Greenwich Mean Time used on the island of Great Britain since 1847. In contrast astronomical GMT began at mean noon, i.e. astronomical day X began at noon of civil day X. The purpose of this was to keep one night's observations under one date. The civil system was adopted as of 0 hours (civil) 1 January 1925. Nautical GMT began 24 hours before astronomical GMT, at least until 1805 in the Royal Navy, but persisted much later elsewhere because it was mentioned at the 1884 conference. In 1884, the Greenwich meridian was used for two-thirds of all charts and maps as their Prime Meridian.[42]
+Among the 41 nations represented at the conference, the advanced time-technologies that had already come into use in Britain were fundamental components of the agreed upon method of arriving at a universal and agreed upon international time. In 1928 Greenwich Mean Time was rebranded for scientific purposes by the International Astronomical Union as Universal Time (UT). This was to avoid confusion with the previous system where the day had begun at noon. As the general public had always begun the day at midnight the timescale continued to be presented to them as Greenwich Mean Time. By 1956, universal time had been split into various versions – UT2, which smoothed for polar motion and seasonal effects, was presented to the public as Greenwich Mean Time. Later, UT1 (which smooths only for polar motion) became the default form of UT used by astronomers and hence the form used in navigation, sunrise and sunset and moonrise and moonset tables where the name Greenwich Mean Time continues to be employed. Greenwich Mean Time is also the preferred method of describing the timescale used by legislators. Even to the present day, UT is still based on an international telescopic system. Observations at the Greenwich Observatory itself ceased in 1954, though the location is still used as the basis for the coordinate system. Because the rotational period of Earth is not perfectly constant, the duration of a second would vary if calibrated to a telescope-based standard like GMT, where the second is defined as 1/86 400 of the mean solar day.
+For the better part of the first century following the "International Meridian Conference," until 1960, the methods and definitions of time-keeping that had been laid out at the conference proved to be adequate to meet time tracking needs of science. Still, with the advent of the "electronic revolution" in the latter half of the 20th century, the technologies that had been available at the time of the Convention of the Metre proved to be in need of further refinement in order to meet the needs of the ever-increasing precision that the "electronic revolution" had begun to require.
+The ephemeris second
+An invariable second (the "ephemeris second") had been defined, use of which removed the errors in ephemerides resulting from the use of the variable mean solar second as the time argument. In 1960 this ephemeris second was made the basis of the "coordinated universal time" which was being derived from atomic clocks. It is a specified fraction of the mean tropical year as at 1900 and, being based on historical telescope observations, corresponds roughly to the mean solar second of the early nineteenth century.[43]
+The SI second
+In 1967 a further step was taken with the introduction of the SI second, essentially the ephemeris second as measured by atomic clocks and formally defined in atomic terms.[44] The SI second (Standard Internationale second) is based directly on the measurement of the atomic-clock observation of the frequency oscillation of caesium atoms. It is the basis of all atomic timescales, e.g. coordinated universal time, GPS time, International Atomic Time, etc. Atomic clocks do not measure nuclear decay rates, which is a common misconception, but rather measure a certain natural vibrational frequency of caesium-133.[45] Coordinated universal time is subject to one constraint which does not affect the other atomic timescales. As it has been adopted as the civil timescale by some countries (most countries have opted to retain mean solar time) it is not permitted to deviate from GMT by more than 0.9 second. This is achieved by the occasional insertion of a leap second.
+Current application of UTC
+Most countries use mean solar time. Australia, Canada (Quebec only), Colombia, France, Germany, New Zealand, Papua New Guinea (Bougainville only), Paraguay, Portugal, Switzerland, the United States and Venezuela use UTC. However, UTC is widely used by the scientific community in countries where mean solar time is official. UTC time is based on the SI second, which was first defined in 1967, and is based on the use of atomic clocks. Some other less used but closely related time-standards include International Atomic Time (TAI), Terrestrial Time, and Barycentric Dynamical Time.
+Between 1967 and 1971, UTC was periodically adjusted by fractional amounts of a second in order to adjust and refine for variations in mean solar time, with which it is aligned. After 1 January 1972, UTC time has been defined as being offset from atomic time by a whole number of seconds, changing only when a leap second is added to keep radio-controlled clocks synchronized with the rotation of the Earth.
+The Global Positioning System also broadcasts a very precise time signal worldwide, along with instructions for converting GPS time to UTC. GPS-time is based on, and regularly synchronized with or from, UTC-time.
+Earth is split up into a number of time zones. Most time zones are exactly one hour apart, and by convention compute their local time as an offset from GMT. For example, time zones at sea are based on GMT. In many locations (but not at sea) these offsets vary twice yearly due to daylight saving time transitions.
+Philosophy
+Main articles: Philosophy of space and time and Temporal finitism
+Time's mortal aspect is personified in this bronze statue by Charles van der Stappen.
+Two distinct viewpoints on time divide many prominent philosophers. One view is that time is part of the fundamental structure of the universe, a dimension in which events occur in sequence. Sir Isaac Newton subscribed to this realist view, and hence it is sometimes referred to as Newtonian time.[16] An opposing view is that time does not refer to any kind of actually existing dimension that events and objects "move through", nor to any entity that "flows", but that it is instead an intellectual concept (together with space and number) that enables humans to sequence and compare events.[56] This second view, in the tradition of Gottfried Leibniz[17] and Immanuel Kant,[18][19] holds that space and time "do not exist in and of themselves, but ... are the product of the way we represent things", because we can know objects only as they appear to us.
+Furthermore, it may be that there is a subjective component to time, but whether or not time itself is "felt", as a sensation, or is a judgment, is a matter of debate.[2][6][7][57][58]
+The Vedas, the earliest texts on Indian philosophy and Hindu philosophy dating back to the late 2nd millennium BC, describe ancient Hindu cosmology, in which the universe goes through repeated cycles of creation, destruction and rebirth, with each cycle lasting 4,320 million years.[59] Ancient Greek philosophers, including Parmenides and Heraclitus, wrote essays on the nature of time.[60] Plato, in the Timaeus, identified time with the period of motion of the heavenly bodies. Aristotle, in Book IV of his Physica defined time as 'number of movement in respect of the before and after'.[61]
+In Book 11 of his Confessions, St. Augustine of Hippo ruminates on the nature of time, asking, "What then is time? If no one asks me, I know: if I wish to explain it to one that asketh, I know not." He begins to define time by what it is not rather than what it is,[62] an approach similar to that taken in other negative definitions. However, Augustine ends up calling time a "distention" of the mind (Confessions 11.26) by which we simultaneously grasp the past in memory, the present by attention, and the future by expectation.
+This view is shared by Abrahamic faiths as they believe time started by creation, therefore the only thing being infinite is God and everything else, including time, is finite.
+Isaac Newton believed in absolute space and absolute time; Leibniz believed that time and space are relational.[63] The differences between Leibniz's and Newton's interpretations came to a head in the famous Leibniz–Clarke correspondence.
+Time is not an empirical concept. For neither co-existence nor succession would be perceived by us, if the representation of time did not exist as a foundation a priori. Without this presupposition we could not represent to ourselves that things exist together at one and the same time, or at different times, that is, contemporaneously, or in succession.
+Immanuel Kant, Critique of Pure Reason (1781), trans. Vasilis Politis (London: Dent., 1991), p.54.
+Immanuel Kant, in the Critique of Pure Reason, described time as an a priori intuition that allows us (together with the other a priori intuition, space) to comprehend sense experience.[64] With Kant, neither space nor time are conceived as substances, but rather both are elements of a systematic mental framework that necessarily structures the experiences of any rational agent, or observing subject. Kant thought of time as a fundamental part of an abstract conceptual framework, together with space and number, within which we sequence events, quantify their duration, and compare the motions of objects. In this view, time does not refer to any kind of entity that "flows," that objects "move through," or that is a "container" for events. Spatial measurements are used to quantify the extent of and distances between objects, and temporal measurements are used to quantify the durations of and between events. Time was designated by Kant as the purest possible schema of a pure concept or category.
+Henri Bergson believed that time was neither a real homogeneous medium nor a mental construct, but possesses what he referred to as Duration. Duration, in Bergson's view, was creativity and memory as an essential component of reality.[65]
+According to Martin Heidegger we do not exist inside time, we are time. Hence, the relationship to the past is a present awareness of having been, which allows the past to exist in the present. The relationship to the future is the state of anticipating a potential possibility, task, or engagement. It is related to the human propensity for caring and being concerned, which causes "being ahead of oneself" when thinking of a pending occurrence. Therefore, this concern for a potential occurrence also allows the future to exist in the present. The present becomes an experience, which is qualitative instead of quantitative. Heidegger seems to think this is the way that a linear relationship with time, or temporal existence, is broken or transcended.[66] We are not stuck in sequential time. We are able to remember the past and project into the future – we have a kind of random access to our representation of temporal existence; we can, in our thoughts, step out of (ecstasis) sequential time.
+Time as "unreal"
+In 5th century BC Greece, Antiphon the Sophist, in a fragment preserved from his chief work On Truth, held that: "Time is not a reality (hypostasis), but a concept (noêma) or a measure (metron)." Parmenides went further, maintaining that time, motion, and change were illusions, leading to the paradoxes of his follower Zeno.[68] Time as an illusion is also a common theme in Buddhist thought.[69][70]
+J. M. E. McTaggart's 1908 The Unreality of Time argues that, since every event has the characteristic of being both present and not present (i.e., future or past), that time is a self-contradictory idea (see also The flow of time).
+These arguments often center on what it means for something to be unreal. Modern physicists generally believe that time is as real as space – though others, such as Julian Barbour in his book The End of Time, argue that quantum equations of the universe take their true form when expressed in the timeless realm containing every possible now or momentary configuration of the universe, called 'platonia' by Barbour.[71]
+A modern philosophical theory called presentism views the past and the future as human-mind interpretations of movement instead of real parts of time (or "dimensions") which coexist with the present. This theory rejects the existence of all direct interaction with the past or the future, holding only the present as tangible. This is one of the philosophical arguments against time travel. This contrasts with eternalism (all time: present, past and future, is real) and the growing block theory (the present and the past are real, but the future is not).
+Physical Definition
+Until Einstein's reinterpretation of the physical concepts associated with time and space, time was considered to be the same everywhere in the universe, with all observers measuring the same time interval for any event.[72] Non-relativistic classical mechanics is based on this Newtonian idea of time.
+Einstein, in his special theory of relativity,[73] postulated the constancy and finiteness of the speed of light for all observers. He showed that this postulate, together with a reasonable definition for what it means for two events to be simultaneous, requires that distances appear compressed and time intervals appear lengthened for events associated with objects in motion relative to an inertial observer.
+The theory of special relativity finds a convenient formulation in Minkowski spacetime, a mathematical structure that combines three dimensions of space with a single dimension of time. In this formalism, distances in space can be measured by how long light takes to travel that distance, e.g., a light-year is a measure of distance, and a meter is now defined in terms of how far light travels in a certain amount of time. Two events in Minkowski spacetime are separated by an invariant interval, which can be either space-like, light-like, or time-like. Events that have a time-like separation cannot be simultaneous in any frame of reference, there must be a temporal component (and possibly a spatial one) to their separation. Events that have a space-like separation will be simultaneous in some frame of reference, and there is no frame of reference in which they do not have a spatial separation. Different observers may calculate different distances and different time intervals between two events, but the invariant interval between the events is independent of the observer (and his or her velocity).
+Classical mechanics
+In non-relativistic classical mechanics, Newton's concept of "relative, apparent, and common time" can be used in the formulation of a prescription for the synchronization of clocks. Events seen by two different observers in motion relative to each other produce a mathematical concept of time that works sufficiently well for describing the everyday phenomena of most people's experience. In the late nineteenth century, physicists encountered problems with the classical understanding of time, in connection with the behavior of electricity and magnetism. Einstein resolved these problems by invoking a method of synchronizing clocks using the constant, finite speed of light as the maximum signal velocity. This led directly to the result that observers in motion relative to one another measure different elapsed times for the same event.
+Spacetime
+Main article: Spacetime
+Time has historically been closely related with space, the two together merging into spacetime in Einstein's special relativity and general relativity. According to these theories, the concept of time depends on the spatial reference frame of the observer, and the human perception as well as the measurement by instruments such as clocks are different for observers in relative motion. For example, if a spaceship carrying a clock flies through space at (very nearly) the speed of light, its crew does not notice a change in the speed of time on board their vessel because everything traveling at the same speed slows down at the same rate (including the clock, the crew's thought processes, and the functions of their bodies). However, to a stationary observer watching the spaceship fly by, the spaceship appears flattened in the direction it is traveling and the clock on board the spaceship appears to move very slowly.
+On the other hand, the crew on board the spaceship also perceives the observer as slowed down and flattened along the spaceship's direction of travel, because both are moving at very nearly the speed of light relative to each other. Because the outside universe appears flattened to the spaceship, the crew perceives themselves as quickly traveling between regions of space that (to the stationary observer) are many light years apart. This is reconciled by the fact that the crew's perception of time is different from the stationary observer's; what seems like seconds to the crew might be hundreds of years to the stationary observer. In either case, however, causality remains unchanged: the past is the set of events that can send light signals to an entity and the future is the set of events to which an entity can send light signals
+Time dilation
+Relativity of simultaneity: Event B is simultaneous with A in the green reference frame, but it occurred before in the blue frame, and occurs later in the red frame.
+Main article: Time dilation
+Einstein showed in his thought experiments that people travelling at different speeds, while agreeing on cause and effect, measure different time separations between events, and can even observe different chronological orderings between non-causally related events. Though these effects are typically minute in the human experience, the effect becomes much more pronounced for objects moving at speeds approaching the speed of light. Subatomic particles exist for a well known average fraction of a second in a lab relatively at rest, but when travelling close to the speed of light they are measured to travel farther and exist for much longer than when at rest. According to the special theory of relativity, in the high-speed particle's frame of reference, it exists, on the average, for a standard amount of time known as its mean lifetime, and the distance it travels in that time is zero, because its velocity is zero. Relative to a frame of reference at rest, time seems to "slow down" for the particle. Relative to the high-speed particle, distances seem to shorten. Einstein showed how both temporal and spatial dimensions can be altered (or "warped") by high-speed motion.
+Einstein (The Meaning of Relativity): "Two events taking place at the points A and B of a system K are simultaneous if they appear at the same instant when observed from the middle point, M, of the interval AB. Time is then defined as the ensemble of the indications of similar clocks, at rest relative to K, which register the same simultaneously."
+Einstein wrote in his book, Relativity, that simultaneity is also relative, i.e., two events that appear simultaneous to an observer in a particular inertial reference frame need not be judged as simultaneous by a second observer in a different inertial frame of reference.
+Relativistic time versus Newtonian time
+Views of spacetime along the world line of a rapidly accelerating observer in a relativistic universe. The events ("dots") that pass the two diagonal lines in the bottom half of the image (the past light cone of the observer in the origin) are the events visible to the observer.
+The animations visualise the different treatments of time in the Newtonian and the relativistic descriptions. At the heart of these differences are the Galilean and Lorentz transformations applicable in the Newtonian and relativistic theories, respectively.
+In the figures, the vertical direction indicates time. The horizontal direction indicates distance (only one spatial dimension is taken into account), and the thick dashed curve is the spacetime trajectory ("world line") of the observer. The small dots indicate specific (past and future) events in spacetime.
+The slope of the world line (deviation from being vertical) gives the relative velocity to the observer. Note how in both pictures the view of spacetime changes when the observer accelerates.
+In the Newtonian description these changes are such that time is absolute:[77] the movements of the observer do not influence whether an event occurs in the 'now' (i.e., whether an event passes the horizontal line through the observer).
+However, in the relativistic description the observability of events is absolute: the movements of the observer do not influence whether an event passes the "light cone" of the observer. Notice that with the change from a Newtonian to a relativistic description, the concept of absolute time is no longer applicable: events move up-and-down in the figure depending on the acceleration of the observer.
+Time appears to have a direction – the past lies behind, fixed and immutable, while the future lies ahead and is not necessarily fixed. Yet for the most part the laws of physics do not specify an arrow of time, and allow any process to proceed both forward and in reverse. This is generally a consequence of time being modelled by a parameter in the system being analysed, where there is no "proper time": the direction of the arrow of time is sometimes arbitrary. Examples of this include the cosmological arrow of time, which points away from the Big Bang, CPT symmetry, and the radiative arrow of time, caused by light only travelling forwards in time (see light cone). In particle physics, the violation of CP symmetry implies that there should be a small counterbalancing time asymmetry to preserve CPT symmetry as stated above. The standard description of measurement in quantum mechanics is also time asymmetric (see Measurement in quantum mechanics). The second law of thermodynamics states that entropy must increase over time (see Entropy). This can be in either direction – Brian Greene theorizes that, according to the equations, the change in entropy occurs symmetrically whether going forward or backward in time. So entropy tends to increase in either direction, and our current low-entropy universe is a statistical aberration, in the similar manner as tossing a coin often enough that eventually heads will result ten times in a row. However, this theory is not supported empirically in local experiment.[78]
+'''] # nopep8
+ ]}
+
+
+class TestLog(TestBase):
+ title = 'Test log'
+
+ @default_test_setup(1, compression=True, optimize_interval=10)
+ async def run(self):
+ await self.client0.connect()
+
+ n = 0
+ for p in DATA.values():
+ n += len(p)
+
+ self.assertEqual(
+ await self.client0.insert(DATA),
+ {'success_msg': 'Successfully inserted {} point(s).'.format(n)})
+
+ await self.test_data()
+
+ # wait for optimization
+ await asyncio.sleep(30)
+
+ await self.test_data()
+
+ self.client0.close()
+
+ # stop and restart
+ result = await self.server0.stop()
+ self.assertTrue(result)
+
+ await self.server0.start(sleep=20)
+
+ await self.client0.connect()
+
+ await self.test_data()
+
+ async def test_data(self):
+ self.assertEqual(
+ await self.client0.query('select * from "log"'),
+ {'log': DATA['log']})
+
+ self.assertEqual(
+ await self.client0.query('select * from "utf16"'),
+ {'utf16': DATA['utf16']})
+
+ self.assertEqual(
+ await self.client0.query('select * from "long_log"'),
+ {'long_log': DATA['long_log']})
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestLog())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+LENPOINTS = 36
+DATA = {
+ 'series-001': [
+ [1471254705000000005, 1.5],
+ [1471254705000000007, -3.5],
+ [1471254705000000010, -7.3]],
+ 'series-002': [
+ [1471254705000000005, 5],
+ [1471254705000000008, -3],
+ [1471254705000000010, -7]],
+ 'series-003': [
+ [1471254705000000005, 10.5],
+ [1471254705000000007, -8.5],
+ [1471254705000000010, -2.7]],
+ 'series-004': [
+ [1471254705000000005, 6],
+ [1471254705000000008, -8],
+ [1471254705000000010, -9]],
+ 'linux-001': [
+ [1471254705000000005, 7.3],
+ [1471254705000000007, -6.4],
+ [1471254705000000010, -9.8]],
+ 'linux-002': [
+ [1471254705000000005, 2],
+ [1471254705000000008, -7],
+ [1471254705000000010, -9]],
+ 'linux-003': [
+ [1471254705000000005, 2.9],
+ [1471254705000000007, -5.7],
+ [1471254705000000010, -0.3]],
+ 'linux-004': [
+ [1471254705000000005, 3],
+ [1471254705000000008, -9],
+ [1471254705000000010, -8]],
+ 'windows-001': [
+ [1471254705000000005, 9.3],
+ [1471254705000000007, -3.3],
+ [1471254705000000010, -1.6]],
+ 'windows-002': [
+ [1471254705000000005, 4],
+ [1471254705000000008, -8],
+ [1471254705000000010, -2]],
+ 'windows-003': [
+ [1471254705000000005, 4.3],
+ [1471254705000000007, -7.9],
+ [1471254705000000010, -1.2]],
+ 'windows-004': [
+ [1471254705000000005, 2],
+ [1471254705000000008, -5],
+ [1471254705000000010, -7]],
+
+}
+
+TIME_PRECISION = 'ns'
+
+
+class TestParenth(TestBase):
+ title = 'Test parentheses'
+
+ @default_test_setup(1, time_precision=TIME_PRECISION)
+ async def run(self):
+ await self.client0.connect()
+
+ self.assertEqual(
+ await self.client0.insert(DATA),
+ {'success_msg': 'Successfully inserted {} point(s).'.format(
+ LENPOINTS)})
+
+ result = await self.client0.query('''
+ list series
+ all - ("series-001" | "series-002" | /windows.*/)
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-003'],
+ ['series-004'],
+ ['linux-001'],
+ ['linux-002'],
+ ['linux-003'],
+ ['linux-004']
+ ])
+ }
+ )
+
+ result = await self.client0.query('''
+ list series all - (
+ "series-001" | "series-002" | (/windows.*/ & /.*001/))
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-003'],
+ ['series-004'],
+ ['linux-001'],
+ ['linux-002'],
+ ['linux-003'],
+ ['linux-004'],
+ ['windows-002'],
+ ['windows-003'],
+ ['windows-004']
+ ])
+ }
+ )
+ result = await self.client0.query('''
+ list series all - (
+ "series-001" | "series-002" | (/windows.*/ - /.*001/))
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-003'],
+ ['series-004'],
+ ['linux-001'],
+ ['linux-002'],
+ ['linux-003'],
+ ['linux-004'],
+ ['windows-001']
+ ])
+ }
+ )
+
+ result = await self.client0.query('''
+ list series (
+ "series-001" | "series-002" | /windows.*/) - /.*003/
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-001'],
+ ['series-002'],
+ ['windows-001'],
+ ['windows-002'],
+ ['windows-004']
+ ])
+ }
+ )
+
+ result = await self.client0.query('''
+ list series all - (/series.*/ ^ /.*001/)
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-001'],
+ ['linux-002'],
+ ['linux-003'],
+ ['linux-004'],
+ ['windows-002'],
+ ['windows-003'],
+ ['windows-004']
+ ])
+ }
+ )
+
+ self.assertEqual(
+ await self.client0.query('''
+ list series (/.*001/ & /linux.*/) - /.*001/
+ '''),
+ {
+ 'columns': ['name'],
+ 'series': []})
+
+ result = await self.client0.query('''
+ list series /.*001/ & (/series.*/ | /linux.*/)
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-001'],
+ ['linux-001']
+ ])
+ }
+ )
+
+ result = await self.client0.query('''
+ list series /.*001/ & ((((/series.*/ | /linux.*/))))
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-001'],
+ ['linux-001']
+ ])
+ }
+ )
+
+ result = await self.client0.query('''
+ list series (/.*001/ | /.*002/) & (/series.*/ | /linux.*/)
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-001'],
+ ['series-002'],
+ ['linux-001'],
+ ['linux-002']
+ ])
+ }
+ )
+
+ result = await self.client0.query('''
+ list series ((/.*001/ | /.*002/) & (/series.*/ | /linux.*/))
+ ''')
+ result['series'] = sorted(result['series'])
+
+ self.assertEqual(
+ result,
+ {
+ 'columns': ['name'],
+ 'series': sorted([
+ ['series-001'],
+ ['series-002'],
+ ['linux-001'],
+ ['linux-002']
+ ])
+ }
+ )
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Query error at position 29. Expecting \*, all, '
+ 'single_quote_str, double_quote_str or \('):
+ await self.client0.query(
+ 'list series /.*/ - {}{}'.format('(' * 10, ')' * 10))
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Memory allocation error or maximum recursion depth reached.'):
+ await self.client0.query('''
+ list series /.*/ -
+ {}/linux.*/{}'''.format('(' * 500, ')' * 500))
+
+ await self.client0.query('alter database set list_limit 5000')
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Limit must be a value between 1 and 5000 '
+ 'but received: 6000.*'):
+ await self.client0.query(
+ 'list series limit 6000')
+
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestParenth())
--- /dev/null
+import os
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import SiriDBAsyncUnixConnection
+from testing import parse_args
+
+
+PIPE_NAME = '/tmp/siridb_pipe_test.sock'
+
+DATA = {
+ 'series num_float': [
+ [1471254705, 1.5],
+ [1471254707, -3.5],
+ [1471254710, -7.3]],
+ 'series num_integer': [
+ [1471254705, 5],
+ [1471254708, -3],
+ [1471254710, -7]],
+ 'series_log': [
+ [1471254710, 'log line one'],
+ [1471254712, 'log line two'],
+ [1471254714, 'another line (three)'],
+ [1471254716, 'and yet one more']]
+}
+
+if os.path.exists(PIPE_NAME):
+ os.unlink(PIPE_NAME)
+
+
+class TestPipeSupport(TestBase):
+ title = 'Test pipe support object'
+
+ @default_test_setup(1, pipe_name=PIPE_NAME)
+ async def run(self):
+
+ pipe_client0 = SiriDBAsyncUnixConnection(PIPE_NAME)
+
+ await pipe_client0.connect('iris', 'siri', self.db.dbname)
+
+ self.assertEqual(
+ await pipe_client0.insert(DATA),
+ {'success_msg': 'Successfully inserted 10 point(s).'})
+
+ self.assertAlmostEqual(
+ await pipe_client0.query('select * from "series num_float"'),
+ {'series num_float': DATA['series num_float']})
+
+ self.assertEqual(
+ await pipe_client0.query('select * from "series num_integer"'),
+ {'series num_integer': DATA['series num_integer']})
+
+ self.assertEqual(
+ await pipe_client0.query('select * from "series_log"'),
+ {'series_log': DATA['series_log']})
+
+ pipe_client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestPipeSupport())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+class TestPool(TestBase):
+ title = 'Test pool object'
+
+ async def insert(self, client, series, n, timeout=1):
+ for _ in range(n):
+ await client.insert_some_series(series, n=0.01, timeout=timeout)
+ await asyncio.sleep(1.0)
+
+ @default_test_setup(4)
+ async def run(self):
+
+ series = gen_series(n=10000)
+
+ await self.client0.connect()
+
+ task0 = asyncio.ensure_future(self.insert(
+ self.client0,
+ series,
+ 200))
+
+ await asyncio.sleep(5)
+
+ await self.db.add_pool(self.server1, sleep=3)
+
+ with self.assertRaises(AssertionError):
+ await self.db.add_pool(self.server2, sleep=3)
+
+ await self.client1.connect()
+ task1 = asyncio.ensure_future(self.insert(
+ self.client1,
+ series,
+ 150))
+
+ await asyncio.sleep(5)
+
+ await self.assertIsRunning(self.db, self.client0, timeout=200)
+
+ await asyncio.sleep(25)
+
+ await self.db.add_replica(self.server3, 1, sleep=3)
+ await self.client3.connect()
+ await self.assertIsRunning(self.db, self.client3, timeout=200)
+
+ await asyncio.sleep(30)
+
+ await self.db.add_pool(self.server2, sleep=3)
+ await self.client2.connect()
+
+ task2 = asyncio.ensure_future(self.insert(
+ self.client2,
+ series,
+ 100))
+
+ await self.assertIsRunning(self.db, self.client0, timeout=600)
+
+ await asyncio.wait_for(task0, None)
+ await asyncio.wait_for(task1, None)
+ await asyncio.wait_for(task2, None)
+
+ await asyncio.sleep(2)
+
+ await self.assertSeries(self.client0, series)
+ await self.assertSeries(self.client1, series)
+ await self.assertSeries(self.client2, series)
+ await self.assertSeries(self.client3, series)
+
+ self.client0.close()
+ self.client1.close()
+ self.client2.close()
+ self.client3.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestPool())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+import math
+import re
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+LENPOINTS = 70
+DATA = {
+ 'series-001 float': [
+ [1471254707, -3.5],
+ [1471254705, 1.5],
+ [1471254710, -7.3]],
+ 'series-001 integer': [
+ [1471254705, 5],
+ [1471254710, -7],
+ [1471254708, -3]],
+ 'series-002 float': [
+ [1471254710, -8.3],
+ [1471254705, 3.5],
+ [1471254707, -2.5]],
+ 'series-002 integer': [
+ [1471254705, 4],
+ [1471254708, -1],
+ [1471254710, -8]],
+ 'aggr': [
+ [1447250868, 530], [1447251168, 520],
+ [1447249033, 531], [1447249337, 534],
+ [1447249633, 535], [1447249937, 531],
+ [1447250249, 532], [1447250549, 537],
+ [1447251449, 54], [1447251749, 54],
+ [1447252049, 513], [1447252349, 537],
+ [1447253244, 533], [1447253549, 538],
+ [1447253849, 534], [1447254149, 532],
+ [1447252649, 528], [1447252968, 531],
+ [1447254449, 533], [1447254748, 537]],
+ 'huge': [
+ [1471254705, 9223372036854775807],
+ [1471254706, 9223372036854775806],
+ [1471254707, 9223372036854775805],
+ [1471254708, 9223372036854775804]],
+ 'equal ts': [
+ [1471254705, 0], [1471254705, 1], [1471254705, 1],
+ [1471254707, 0], [1471254707, 1], [1471254708, 0],
+ ],
+ 'variance': [
+ [1471254705, 2.75], [1471254706, 1.75], [1471254707, 1.25],
+ [1471254708, 0.25], [1471254709, 0.5], [1471254710, 1.25],
+ [1471254711, 3.5]
+ ],
+ 'pvariance': [
+ [1471254705, 0.0], [1471254706, 0.25], [1471254707, 0.25],
+ [1471254708, 1.25], [1471254709, 1.5], [1471254710, 1.75],
+ [1471254711, 2.75], [1471254712, 3.25]
+ ],
+ 'filter': [
+ [1471254705, 5],
+ [1471254710, -3],
+ [1471254715, -7],
+ [1471254720, 7]
+ ],
+ 'one': [
+ [1471254710, 1]
+ ],
+ 'log': [
+ [1471254710, 'log line one'],
+ [1471254712, 'log line two'],
+ [1471254714, 'another line (three)'],
+ [1471254716, 'and yet one more'],
+ ],
+ 'special': [
+ [1471254705, 0.1],
+ [1471254706, math.nan],
+ [1471254707, math.inf],
+ [1471254708, -math.inf],
+ ]
+}
+
+
+class TestSelect(TestBase):
+ title = 'Test select and aggregate functions'
+
+ @default_test_setup(1, compression=False, buffer_size=1024)
+ async def run(self):
+ await self.client0.connect()
+
+ self.assertEqual(
+ await self.client0.insert(DATA),
+ {'success_msg': 'Successfully inserted {} point(s).'.format(
+ LENPOINTS)})
+
+ for name in DATA:
+ DATA[name] = sorted(DATA[name])
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference() from "series-001 integer"'),
+ {'series-001 integer': [[1471254708, -8], [1471254710, -4]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference() => difference() '
+ 'from "series-001 integer"'),
+ {'series-001 integer': [[1471254710, 4]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference() => difference() => difference() '
+ 'from "series-001 integer"'),
+ {'series-001 integer': []})
+
+ now = int(time.time())
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference({}) from "series-001 integer"'.format(now)),
+ {'series-001 integer': [[now, -12]]})
+
+ now = int(time.time())
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference({}) from "series-001 integer"'.format(now)),
+ {'series-001 integer': [[now, -12]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select * from /series-001.*/ '
+ 'merge as "median_low" using median_low({})'
+ .format(now)),
+ {'median_low': [[now, -3.5]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select * from /series-001.*/ '
+ 'merge as "median_high" using median_high({})'
+ .format(now)),
+ {'median_high': [[now, -3.0]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select * from /series.*/ '
+ 'merge as "max" using max(1s)'),
+ {'max': [
+ [1471254705, 5.0],
+ [1471254707, -2.5],
+ [1471254708, -1.0],
+ [1471254710, -7.0]
+ ]})
+
+ # Test all aggregation methods
+
+ self.assertEqual(
+ await self.client0.query('select sum(1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 2663], [1447254000, 5409], [1447257600, 1602]]})
+
+ self.assertEqual(
+ await self.client0.query('select count(1h) from "aggr"'),
+ {'aggr': [[1447250400, 5], [1447254000, 12], [1447257600, 3]]})
+
+ self.assertEqual(
+ await self.client0.query('select mean(1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 532.6],
+ [1447254000, 450.75],
+ [1447257600, 534.0]]})
+
+ self.assertEqual(
+ await self.client0.query('select median(1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 532.0],
+ [1447254000, 530.5],
+ [1447257600, 533.0]]})
+
+ self.assertEqual(
+ await self.client0.query('select median_low(1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 532], [1447254000, 530], [1447257600, 533]]})
+
+ self.assertEqual(
+ await self.client0.query('select median_high(1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 532], [1447254000, 531], [1447257600, 533]]})
+
+ self.assertEqual(
+ await self.client0.query('select min(1h) from "aggr"'),
+ {'aggr': [[1447250400, 531], [1447254000, 54], [1447257600, 532]]})
+
+ self.assertEqual(
+ await self.client0.query('select max(1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 535], [1447254000, 538], [1447257600, 537]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select variance(1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 3.3],
+ [1447254000, 34396.931818181816],
+ [1447257600, 7.0]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select pvariance(1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 2.6399999999999997],
+ [1447254000, 31530.520833333332],
+ [1447257600, 4.666666666666667]]})
+
+ self.assertEqual(
+ await self.client0.query('select * from ({}) - ("a", "b")'.format(
+ ','.join(['"aggr"'] * 600)
+ )),
+ {'aggr': DATA['aggr']}
+ )
+
+ self.assertEqual(
+ await self.client0.query('select difference(1h) from "aggr"'),
+ {'aggr': [[1447250400, 1], [1447254000, -3], [1447257600, 5]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select derivative(1, 1h) from "aggr"'),
+ {'aggr': [
+ [1447250400, 0.0002777777777777778],
+ [1447254000, -0.0008333333333333333],
+ [1447257600, 0.001388888888888889]]})
+
+ self.assertEqual(
+ await self.client0.query('select filter(>534) from "aggr"'),
+ {'aggr': [
+ [1447249633, 535],
+ [1447250549, 537],
+ [1447252349, 537],
+ [1447253549, 538],
+ [1447254748, 537]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(/l.*/) from * where type == string'),
+ {'log': [p for p in DATA['log'] if re.match('l.*', p[1])]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(==/l.*/) from * where type == string'),
+ {'log': [p for p in DATA['log'] if re.match('l.*', p[1])]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(!=/l.*/) from * where type == string'),
+ {'log': [p for p in DATA['log'] if not re.match('l.*', p[1])]})
+
+ self.assertEqual(
+ await self.client0.query('select limit(300, mean) from "aggr"'),
+ {'aggr': DATA['aggr']})
+
+ self.assertEqual(
+ await self.client0.query('select limit(1, sum) from "aggr"'),
+ {'aggr': [[1447254748, 9674]]})
+
+ self.assertEqual(
+ await self.client0.query('select limit(3, mean) from "aggr"'),
+ {'aggr': [
+ [1447250938, 532.8571428571429],
+ [1447252844, 367.6666666666667],
+ [1447254750, 534.0]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select limit(2, max) from "series-001 float"'),
+ {'series-001 float': [[1471254707, 1.5], [1471254713, -7.3]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select variance(1471254712) from "variance"'),
+ {'variance': [[1471254712, 1.3720238095238095]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select pvariance(1471254715) from "pvariance"'),
+ {'pvariance': [[1471254715, 1.25]]})
+
+ self.assertEqual(
+ await self.client0.query('select * from "one"'),
+ {'one': [[1471254710, 1]]})
+
+ self.assertEqual(
+ await self.client0.query('select * from "log"'),
+ {'log': DATA['log']})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(~"log") => filter(!~"one") from "log"'),
+ {'log': [DATA['log'][1]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(!=nan) from "special"'),
+ {'special': [p for p in DATA['special'] if not math.isnan(p[1])]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(==nan) from "special"'),
+ {'special': [p for p in DATA['special'] if math.isnan(p[1])]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(>=nan) from "special"'),
+ {'special': [p for p in DATA['special'] if math.isnan(p[1])]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(<=nan) from "special"'),
+ {'special': [p for p in DATA['special'] if math.isnan(p[1])]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(==inf) from "special"'),
+ {'special': [p for p in DATA['special'] if p[1] == math.inf]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(<inf) from "special"'),
+ {'special': [p for p in DATA['special'] if p[1] < math.inf]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(>inf) from "special"'),
+ {'special': []})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(==-inf) from "special"'),
+ {'special': [p for p in DATA['special'] if p[1] == -math.inf]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(>-inf) from "special"'),
+ {'special': [p for p in DATA['special'] if p[1] > -math.inf]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(<-inf) from "special"'),
+ {'special': []})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(~"one") prefix "1-", '
+ 'filter(~"two") prefix "2-" from "log"'),
+ {
+ '1-log': [
+ [1471254710, 'log line one'],
+ [1471254716, 'and yet one more']],
+ '2-log': [[1471254712, 'log line two']]
+ })
+
+ self.assertEqual(
+ await self.client0.query('select timeval() from "aggr"'),
+ {'aggr': [
+ [1447249033, 1447249033],
+ [1447249337, 1447249337],
+ [1447249633, 1447249633],
+ [1447249937, 1447249937],
+ [1447250249, 1447250249],
+ [1447250549, 1447250549],
+ [1447250868, 1447250868],
+ [1447251168, 1447251168],
+ [1447251449, 1447251449],
+ [1447251749, 1447251749],
+ [1447252049, 1447252049],
+ [1447252349, 1447252349],
+ [1447252649, 1447252649],
+ [1447252968, 1447252968],
+ [1447253244, 1447253244],
+ [1447253549, 1447253549],
+ [1447253849, 1447253849],
+ [1447254149, 1447254149],
+ [1447254449, 1447254449],
+ [1447254748, 1447254748]]})
+
+ self.assertEqual(
+ await self.client0.query('select interval() from "aggr"'),
+ {'aggr': [
+ [1447249337, 304],
+ [1447249633, 296],
+ [1447249937, 304],
+ [1447250249, 312],
+ [1447250549, 300],
+ [1447250868, 319],
+ [1447251168, 300],
+ [1447251449, 281],
+ [1447251749, 300],
+ [1447252049, 300],
+ [1447252349, 300],
+ [1447252649, 300],
+ [1447252968, 319],
+ [1447253244, 276],
+ [1447253549, 305],
+ [1447253849, 300],
+ [1447254149, 300],
+ [1447254449, 300],
+ [1447254748, 299]]})
+
+ self.assertEqual(
+ await self.client0.query('select difference() from "one"'),
+ {'one': []})
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Regular expressions can only be used with.*'):
+ await self.client0.query('select filter(~//) from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Cannot use a string filter on number type.'):
+ await self.client0.query('select filter(//) from "aggr"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use mean\(\) on string type\.'):
+ await self.client0.query('select mean(1w) from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Group by time must be an integer value larger than zero\.'):
+ await self.client0.query('select mean(0) from "aggr"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Limit must be an integer value larger than zero\.'):
+ await self.client0.query('select limit(6 - 6, mean) from "aggr"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use a string filter on number type\.'):
+ await self.client0.query(
+ 'select * from "aggr" '
+ 'merge as "t" using filter("0")')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use difference\(\) on string type\.'):
+ await self.client0.query('select difference() from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use derivative\(\) on string type\.'):
+ await self.client0.query('select derivative(6, 3) from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use derivative\(\) on string type\.'):
+ await self.client0.query('select derivative() from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Overflow detected while using sum\(\)\.'):
+ await self.client0.query('select sum(now) from "huge"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Max depth reached in \'where\' expression!'):
+ await self.client0.query(
+ 'select * from "aggr" where ((((((length > 1))))))')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Cannot compile regular expression.*'):
+ await self.client0.query(
+ 'select * from /(bla/')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Memory allocation error or maximum recursion depth reached.'):
+ await self.client0.query(
+ 'select * from {}"aggr"{}'.format(
+ '(' * 501,
+ ')' * 501))
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Query too long.'):
+ await self.client0.query('select * from "{}"'.format('a' * 65535))
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Error while merging points. Make sure the destination '
+ 'series name is valid.'):
+ await self.client0.query(
+ 'select * from "aggr", "huge" merge as ""')
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select min(2h) prefix "min-", max(1h) prefix "max-" '
+ 'from /.*/ where type == integer and name != "filter" '
+ 'and name != "one" and name != "series-002 integer" '
+ 'merge as "int_min_max" using median_low(1) => difference()'),
+ {
+ 'max-int_min_max': [
+ [1447254000, 3], [1447257600, -1], [1471255200, -532]],
+ 'min-int_min_max': [
+ [1447257600, -477], [1471255200, -54]]})
+
+ await self.client0.query('select derivative() from "equal ts"')
+
+ self.assertEqual(
+ await self.client0.query('select first() from *'),
+ {k: [v[0]] for k, v in DATA.items()})
+
+ self.assertEqual(
+ await self.client0.query('select last() from *'),
+ {k: [v[-1]] for k, v in DATA.items()})
+
+ self.assertEqual(
+ await self.client0.query('select count() from *'),
+ {k: [[v[-1][0], len(v)]] for k, v in DATA.items()})
+
+ self.assertEqual(
+ await self.client0.query('select mean() from "aggr"'),
+ {'aggr': [[
+ DATA['aggr'][-1][0],
+ sum([x[1] for x in DATA['aggr']]) / len(DATA['aggr'])]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select stddev() from "aggr"'),
+ {'aggr': [[
+ DATA['aggr'][-1][0],
+ 147.07108914792838]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select stddev(1h) from "aggr"'),
+ {"aggr": [
+ [1447250400, 1.8165902124584952],
+ [1447254000, 185.46409846162092],
+ [1447257600, 2.6457513110645907]]})
+
+ # test prefix, suffex
+ result = await self.client0.query(
+ 'select sum(1d) prefix "sum-" suffix "-sum", '
+ 'min(1d) prefix "minimum-", '
+ 'max(1d) suffix "-maximum" from "aggr"')
+
+ self.assertIn('sum-aggr-sum', result)
+ self.assertIn('minimum-aggr', result)
+ self.assertIn('aggr-maximum', result)
+
+ await self.client0.query('alter database set select_points_limit 10')
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Query has reached the maximum number of selected points.*'):
+ await self.client0.query(
+ 'select * from /.*/')
+ await self.client0.query(
+ 'alter database set select_points_limit 1000000')
+
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestSelect())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+import math
+import re
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+LENPOINTS = 70
+DATA = {
+ 'series-001 float': [
+ [1471254705000000005, 1.5],
+ [1471254705000000007, -3.5],
+ [1471254705000000010, -7.3]],
+ 'series-001 integer': [
+ [1471254705000000005, 5],
+ [1471254705000000008, -3],
+ [1471254705000000010, -7]],
+ 'series-002 float': [
+ [1471254705000000005, 3.5],
+ [1471254705000000007, -2.5],
+ [1471254705000000010, -8.3]],
+ 'series-002 integer': [
+ [1471254705000000005, 4],
+ [1471254705000000008, -1],
+ [1471254705000000010, -8]],
+ 'aggr': [
+ [1447249049033000000, 531], [1447249049337000000, 534],
+ [1447249049633000000, 535], [1447249049937000000, 531],
+ [1447249050249000000, 532], [1447249050549000000, 537],
+ [1447249050868000000, 530], [1447249051168000000, 520],
+ [1447249051449000000, 54], [1447249051749000000, 54],
+ [1447249052049000000, 513], [1447249052349000000, 537],
+ [1447249052649000000, 528], [1447249052968000000, 531],
+ [1447249053244000000, 533], [1447249053549000000, 538],
+ [1447249053849000000, 534], [1447249054149000000, 532],
+ [1447249054449000000, 533], [1447249054748000000, 537]],
+
+ 'huge': [
+ [1471254705000000005, 9223372036854775807],
+ [1471254705000000006, 9223372036854775806],
+ [1471254705000000007, 9223372036854775805],
+ [1471254705000000008, 9223372036854775804]],
+ 'equal ts': [
+ [1471254705000000005, 0], [1471254705000000005, 1],
+ [1471254705000000005, 1], [1471254705000000007, 0],
+ [1471254705000000007, 1], [1471254705000000007, 0],
+ ],
+ 'variance': [
+ [1471254705000000005, 2.75], [1471254705000000006, 1.75],
+ [1471254705000000007, 1.25], [1471254705000000008, 0.25],
+ [1471254705000000009, 0.5], [1471254705000000010, 1.25],
+ [1471254705000000011, 3.5]
+ ],
+ 'pvariance': [
+ [1471254705000000005, 0.0], [1471254705000000006, 0.25],
+ [1471254705000000007, 0.25], [1471254705000000008, 1.25],
+ [1471254705000000009, 1.5], [1471254705000000010, 1.75],
+ [1471254705000000011, 2.75], [1471254705000000012, 3.25]
+ ],
+ 'filter': [
+ [1471254705000000005, 5],
+ [1471254705000000010, -3],
+ [1471254705000000015, -7],
+ [1471254705000000020, 7]
+ ],
+ 'one': [
+ [1471254705000000010, 1]
+ ],
+ 'log': [
+ [1471254705000000010, 'log line one'],
+ [1471254705000000012, 'log line two'],
+ [1471254705000000014, 'another line (three)'],
+ [1471254705000000016, 'and yet one more'],
+ ],
+ 'special': [
+ [1471254705000000005, 0.1],
+ [1471254705000000006, math.nan],
+ [1471254705000000007, math.inf],
+ [1471254705000000008, -math.inf],
+ ]
+}
+
+
+TIME_PRECISION = 'ns'
+
+
+class TestSelectNano(TestBase):
+ title = 'Test select and aggregate functions (ns)'
+
+ GEN_POINTS = functools.partial(
+ gen_points, n=1, time_precision=TIME_PRECISION)
+
+ @default_test_setup(1, time_precision=TIME_PRECISION)
+ async def run(self):
+ await self.client0.connect()
+
+ self.assertEqual(
+ await self.client0.insert(DATA),
+ {'success_msg': 'Successfully inserted {} point(s).'.format(
+ LENPOINTS)})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference() from "series-001 integer"'),
+ {'series-001 integer': [
+ [1471254705000000008, -8],
+ [1471254705000000010, -4]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference() => difference() '
+ 'from "series-001 integer"'),
+ {'series-001 integer': [[1471254705000000010, 4]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference() => difference() => difference() '
+ 'from "series-001 integer"'),
+ {'series-001 integer': []})
+
+ now = int(time.time()*1000000000)
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference({}) from "series-001 integer"'.format(now)),
+ {'series-001 integer': [[now, -12]]})
+
+ now = int(time.time()*1000000000)
+ self.assertEqual(
+ await self.client0.query(
+ 'select difference({}) from "series-001 integer"'.format(now)),
+ {'series-001 integer': [[now, -12]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select * from /series-001.*/ '
+ 'merge as "median_low" using median_low({})'
+ .format(now)),
+ {'median_low': [[now, -3.5]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select * from /series-001.*/ '
+ 'merge as "median_high" using median_high({})'
+ .format(now)),
+ {'median_high': [[now, -3.0]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select max(1) from /series.*/ '
+ 'merge as "max" using max(1)'),
+ {'max': [
+ [1471254705000000005, 5.0],
+ [1471254705000000007, -2.5],
+ [1471254705000000008, -1.0],
+ [1471254705000000010, -7.0]
+ ]})
+
+ # Test all aggregation methods
+
+ self.assertEqual(
+ await self.client0.query('select sum(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 2131],
+ [1447249051000000000, 1599],
+ [1447249052000000000, 628],
+ [1447249053000000000, 2109],
+ [1447249054000000000, 1605],
+ [1447249055000000000, 1602]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select count(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 4],
+ [1447249051000000000, 3],
+ [1447249052000000000, 3],
+ [1447249053000000000, 4],
+ [1447249054000000000, 3],
+ [1447249055000000000, 3]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select mean(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 532.75],
+ [1447249051000000000, 533.0],
+ [1447249052000000000, 209.3333333333333333334],
+ [1447249053000000000, 527.25],
+ [1447249054000000000, 535.0],
+ [1447249055000000000, 534.0]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select median(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 532.5],
+ [1447249051000000000, 532.0],
+ [1447249052000000000, 54.0],
+ [1447249053000000000, 529.5],
+ [1447249054000000000, 534.0],
+ [1447249055000000000, 533.0]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select median_low(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 531.0],
+ [1447249051000000000, 532.0],
+ [1447249052000000000, 54.0],
+ [1447249053000000000, 528.0],
+ [1447249054000000000, 534.0],
+ [1447249055000000000, 533.0]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select median_high(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 534.0],
+ [1447249051000000000, 532.0],
+ [1447249052000000000, 54.0],
+ [1447249053000000000, 531.0],
+ [1447249054000000000, 534.0],
+ [1447249055000000000, 533.0]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select min(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 531.0],
+ [1447249051000000000, 530.0],
+ [1447249052000000000, 54.0],
+ [1447249053000000000, 513.0],
+ [1447249054000000000, 533.0],
+ [1447249055000000000, 532.0]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select max(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 535.0],
+ [1447249051000000000, 537.0],
+ [1447249052000000000, 520.0],
+ [1447249053000000000, 537.0],
+ [1447249054000000000, 538.0],
+ [1447249055000000000, 537.0]
+ ]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select variance(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 4.25],
+ [1447249051000000000, 13.0],
+ [1447249052000000000, 72385.3333333333333333333],
+ [1447249053000000000, 104.25],
+ [1447249054000000000, 7.0],
+ [1447249055000000000, 7.0]
+ ]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select pvariance(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 3.1875],
+ [1447249051000000000, 8.666666666666666667],
+ [1447249052000000000, 48256.8888888888888888887],
+ [1447249053000000000, 78.1875],
+ [1447249054000000000, 4.666666666666666667],
+ [1447249055000000000, 4.666666666666666667]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select difference(1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 0],
+ [1447249051000000000, -2],
+ [1447249052000000000, -466],
+ [1447249053000000000, 18],
+ [1447249054000000000, 1],
+ [1447249055000000000, 5]
+ ]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select derivative(1, 1s) from "aggr"'),
+ {'aggr': [
+ [1447249050000000000, 0.0],
+ [1447249051000000000, -0.000000002],
+ [1447249052000000000, -0.000000466],
+ [1447249053000000000, 0.000000018],
+ [1447249054000000000, 0.000000001],
+ [1447249055000000000, 0.000000005]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query('select filter(>534) from "aggr"'),
+ {'aggr': [
+ [1447249049633000000, 535],
+ [1447249050549000000, 537],
+ [1447249052349000000, 537],
+ [1447249053549000000, 538],
+ [1447249054748000000, 537]
+ ]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(/l.*/) from * where type == string'),
+ {'log': [p for p in DATA['log'] if re.match('l.*', p[1])]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(==/l.*/) from * where type == string'),
+ {'log': [p for p in DATA['log'] if re.match('l.*', p[1])]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(!=/l.*/) from * where type == string'),
+ {'log': [p for p in DATA['log'] if not re.match('l.*', p[1])]})
+
+ self.assertEqual(
+ await self.client0.query('select limit(300, mean) from "aggr"'),
+ {'aggr': DATA['aggr']})
+
+ self.assertEqual(
+ await self.client0.query('select limit(1, sum) from "aggr"'),
+ {'aggr': [[1447249054748000000, 9674]]})
+
+ # The interval over which the mean is calculated is obtained
+ # by dividing the time period of the serie by the max_points
+ # and adding 1 ns to include the last point of the series.
+ self.assertEqual(
+ await self.client0.query('select limit(3, mean) from "aggr"'),
+ {'aggr': [
+ [1447249050938000000, 532.8571428571429],
+ [1447249052843000001, 367.6666666666667],
+ [1447249054748000002, 534.0]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select limit(2, max) from "series-001 float"'),
+ {'series-001 float': [
+ [1471254705000000007, 1.5],
+ [1471254705000000013, -7.3]
+ ]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select variance(1471254705000000012) from "variance"'),
+ {'variance': [[1471254705000000012, 1.3720238095238095]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select pvariance(1471254705000000015) from "pvariance"'),
+ {'pvariance': [[1471254705000000015, 1.25]]})
+
+ self.assertEqual(
+ await self.client0.query('select * from "one"'),
+ {'one': [[1471254705000000010, 1]]})
+
+ self.assertEqual(
+ await self.client0.query('select * from "log"'),
+ {'log': DATA['log']})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(~"log") => filter(!~"one") from "log"'),
+ {'log': [DATA['log'][1]]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(!=nan) from "special"'),
+ {'special': [p for p in DATA['special'] if not math.isnan(p[1])]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(==nan) from "special"'),
+ {'special': [p for p in DATA['special'] if math.isnan(p[1])]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(>=nan) from "special"'),
+ {'special': [p for p in DATA['special'] if math.isnan(p[1])]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(<=nan) from "special"'),
+ {'special': [p for p in DATA['special'] if math.isnan(p[1])]})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(==inf) from "special"'),
+ {'special': [p for p in DATA['special'] if p[1] == math.inf]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(<inf) from "special"'),
+ {'special': [p for p in DATA['special'] if p[1] < math.inf]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(>inf) from "special"'),
+ {'special': []})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(==-inf) from "special"'),
+ {'special': [p for p in DATA['special'] if p[1] == -math.inf]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(>-inf) from "special"'),
+ {'special': [p for p in DATA['special'] if p[1] > -math.inf]})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select filter(<-inf) from "special"'),
+ {'special': []})
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select filter(~"one") prefix "1-", '
+ 'filter(~"two") prefix "2-" from "log"'),
+ {
+ '1-log': [
+ [1471254705000000010, 'log line one'],
+ [1471254705000000016, 'and yet one more']],
+ '2-log': [[1471254705000000012, 'log line two']]
+ })
+
+ self.assertEqual(
+ await self.client0.query('select difference() from "one"'),
+ {'one': []})
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Regular expressions can only be used with.*'):
+ await self.client0.query('select filter(~//) from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Cannot use a string filter on number type.'):
+ await self.client0.query('select filter(//) from "aggr"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use mean\(\) on string type\.'):
+ await self.client0.query('select mean(1w) from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Group by time must be an integer value larger than zero\.'):
+ await self.client0.query('select mean(0) from "aggr"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Limit must be an integer value larger than zero\.'):
+ await self.client0.query('select limit(6 - 6, mean) from "aggr"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use a string filter on number type\.'):
+ await self.client0.query(
+ 'select * from "aggr" '
+ 'merge as "t" using filter("0")')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use difference\(\) on string type\.'):
+ await self.client0.query('select difference() from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use derivative\(\) on string type\.'):
+ await self.client0.query('select derivative(6, 3) from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Cannot use derivative\(\) on string type\.'):
+ await self.client0.query('select derivative() from "log"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ r'Overflow detected while using sum\(\)\.'):
+ await self.client0.query('select sum(now) from "huge"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Max depth reached in \'where\' expression!'):
+ await self.client0.query(
+ 'select * from "aggr" where ((((((length > 1))))))')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Cannot compile regular expression.*'):
+ await self.client0.query(
+ 'select * from /(bla/')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Memory allocation error or maximum recursion depth reached.'):
+ await self.client0.query(
+ 'select * from {}"aggr"{}'.format(
+ '(' * 501,
+ ')' * 501))
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Query too long.'):
+ await self.client0.query('select * from "{}"'.format('a' * 65535))
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Error while merging points. Make sure the destination '
+ 'series name is valid.'):
+ await self.client0.query(
+ 'select * from "aggr", "huge" merge as ""')
+
+ self.assertEqual(
+ await self.client0.query(
+ 'select min(2s) prefix "min-", max(1s) prefix "max-" '
+ 'from /.*/ where type == integer and name != "filter" '
+ 'and name != "one" and name != "series-002 integer" '
+ 'merge as "int_min_max" using median_low(1) => difference()'),
+ {
+ 'max-int_min_max': [
+ [1447249051000000000, 2],
+ [1447249052000000000, -17],
+ [1447249053000000000, 17],
+ [1447249054000000000, 1],
+ [1447249055000000000, -1],
+ [1471254706000000000, -532]
+ ],
+ 'min-int_min_max': [
+ [1447249052000000000, -477],
+ [1447249054000000000, 459],
+ [1447249056000000000, 19],
+ [1471254706000000000, -532]
+ ]})
+
+ await self.client0.query('select derivative() from "equal ts"')
+
+ self.assertEqual(
+ await self.client0.query('select first() from *'),
+ {k: [v[0]] for k, v in DATA.items()})
+
+ self.assertEqual(
+ await self.client0.query('select last() from *'),
+ {k: [v[-1]] for k, v in DATA.items()})
+
+ self.assertEqual(
+ await self.client0.query('select count() from *'),
+ {k: [[v[-1][0], len(v)]] for k, v in DATA.items()})
+
+ self.assertEqual(
+ await self.client0.query('select mean() from "aggr"'),
+ {'aggr': [[
+ DATA['aggr'][-1][0],
+ sum([x[1] for x in DATA['aggr']]) / len(DATA['aggr'])]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select stddev() from "aggr"'),
+ {'aggr': [[
+ DATA['aggr'][-1][0],
+ 147.07108914792838]]})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select stddev(1s) from "aggr"'),
+ {"aggr": [
+ [1447249050000000000, 2.0615528128088333333],
+ [1447249051000000000, 3.6055512754639999999],
+ [1447249052000000000, 269.0452254423666666667],
+ [1447249053000000000, 10.2102889283311111111],
+ [1447249054000000000, 2.6457513110645999999],
+ [1447249055000000000, 2.6457513110645999999]
+ ]})
+
+ # test prefix, suffex
+ result = await self.client0.query(
+ 'select sum(1d) prefix "sum-" suffix "-sum", '
+ 'min(1d) prefix "minimum-", '
+ 'max(1d) suffix "-maximum" from "aggr"')
+
+ self.assertIn('sum-aggr-sum', result)
+ self.assertIn('minimum-aggr', result)
+ self.assertIn('aggr-maximum', result)
+
+ await self.client0.query('alter database set select_points_limit 10')
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Query has reached the maximum number of selected points.*'):
+ await self.client0.query(
+ 'select * from /.*/')
+ await self.client0.query(
+ 'alter database set select_points_limit 1000000')
+
+ self.client0.close()
+
+ # return False
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestSelectNano())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+PI = 'ԉ'
+Klingon = ' ' + \
+ 'qajunpaQHeylIjmo’ batlh DuSuvqang charghwI’ ‘It.'
+
+data = {
+ 'string': [
+ [1538660000, "some string value"],
+ [1538660010, -123456789],
+ [1538660020, -0.5],
+ [1538660030, 1/3],
+ ],
+ 'integer': [
+ [1538660000, 1],
+ [1538660010, 35.6],
+ [1538660020, "-50%"],
+ [1538660030, ""],
+ [1538660035, "garbage"],
+ [1538660040, "18446744073709551616"],
+ [1538660050, "-18446744073709551616"],
+ ],
+ 'double': [
+ [1538660000, 1.0],
+ [1538660010, -35],
+ [1538660010, "-50%"],
+ [1538660030, ""],
+ [1538660035, "garbage"],
+ ]
+}
+
+expected = {
+ 'string': [
+ [1538660000, "some string value"],
+ [1538660010, '-123456789'],
+ [1538660020, '-0,500000'],
+ [1538660030, '0,333333'],
+ ],
+ 'integer': [
+ [1538660000, 1],
+ [1538660010, 35],
+ [1538660020, -50],
+ [1538660030, 0],
+ [1538660035, 0],
+ [1538660040, 9223372036854775807],
+ [1538660050, -9223372036854775808],
+ ],
+ 'double': [
+ [1538660000, 1.0],
+ [1538660010, -35.0],
+ [1538660010, -50.0],
+ [1538660030, 0.0],
+ [1538660035, 0.0],
+ ]
+}
+
+
+class TestSeries(TestBase):
+ title = 'Test series object'
+
+ @default_test_setup(1)
+ async def run(self):
+ await self.client0.connect()
+
+ points = gen_points(n=10)
+
+ self.assertEqual(
+ await self.client0.insert({
+ PI: points,
+ Klingon: points
+ }), {'success_msg': 'Successfully inserted 20 point(s).'})
+
+ self.assertEqual(
+ await self.client0.query('select * from "{}"'.format(PI)),
+ {PI: sorted(points)})
+
+ self.assertEqual(
+ await self.client0.query('select * from "{}"'.format(Klingon)),
+ {Klingon: sorted(points)})
+
+ self.assertEqual(
+ await self.client0.insert(data),
+ {'success_msg': 'Successfully inserted 16 point(s).'})
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select * from "string", "integer", "double"'),
+ expected)
+
+ self.assertAlmostEqual(
+ await self.client0.query(
+ 'select * from "x", "string", "integer", "double", "nexist"'),
+ expected)
+
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestSeries())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+class TestServer(TestBase):
+ title = 'Test server object'
+
+ Server.SERVER_ADDRESS = 'localhost'
+ Server.IP_SUPPORT = 'IPV4ONLY'
+
+ @default_test_setup(4)
+ async def run(self):
+
+ await self.client0.connect()
+
+ await self.db.add_pool(self.server1)
+ await self.assertIsRunning(self.db, self.client0, timeout=20)
+ await asyncio.sleep(5)
+
+ await self.client1.connect()
+
+ for port in (9010, 9011):
+ result = await self.client0.query(
+ 'alter server "localhost:{}" set log_level error'.format(port))
+ self.assertEqual(
+ result.pop('success_msg'),
+ "Successfully set log level to 'error' on 'localhost:{}'."
+ .format(port))
+
+ result = await self.client1.query('list servers log_level')
+ self.assertEqual(result.pop('servers'), [['error'], ['error']])
+
+ result = await self.client1.query('list servers uuid')
+
+ for uuid in result.pop('servers'):
+ result = await self.client0.query(
+ 'alter server {} set log_level debug'.format(uuid[0]))
+
+ result = await self.client1.query('list servers log_level')
+ self.assertEqual(result.pop('servers'), [['debug'], ['debug']])
+
+ result = await self.client0.query('alter servers set log_level info')
+ self.assertEqual(
+ result.pop('success_msg'),
+ "Successfully set log level to 'info' on 2 servers.")
+
+ result = await self.client1.query('list servers log_level')
+ self.assertEqual(result.pop('servers'), [['info'], ['info']])
+
+ result = await self.client1.query(
+ 'list servers active_tasks where active_tasks == 1 and '
+ 'idle_time >= 0 and idle_percentage <= 100')
+ self.assertEqual(result.pop('servers'), [[1], [1]])
+
+ result = await self.client0.query(
+ 'alter servers where active_handles > 1 set log_level debug')
+
+ result = await self.client1.query('list servers log_level')
+ self.assertEqual(result.pop('servers'), [['debug'], ['debug']])
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "Query error at position 42. Expecting "
+ "debug, info, warning, error or critical"):
+ await self.client0.query(
+ 'alter server "localhost:{}" set log_level unknown')
+
+ self.client1.close()
+ result = await self.server1.stop()
+ self.assertTrue(result)
+
+ self.server1.listen_backend_port = 9111
+ self.server1.create()
+ await self.server1.start(sleep=20)
+
+ await asyncio.sleep(35)
+
+ result = await self.client0.query('list servers status')
+ self.assertEqual(result.pop('servers'), [['running'], ['running']])
+
+ await self.client1.connect()
+ result = await self.client1.query('show server')
+ self.assertEqual(result.pop('data'), [
+ {'name': 'server', 'value': 'localhost:9111'}])
+
+ await self.db.add_replica(self.server2, 1)
+ await self.assertIsRunning(self.db, self.client0, timeout=35)
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "Cannot remove server 'localhost:9010' "
+ "because this is the only server for pool 0"):
+ await self.client1.query('drop server "localhost:9010"')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "Cannot remove server 'localhost:9012' "
+ "because the server is still online.*"):
+ await self.client1.query('drop server "localhost:9012"')
+
+ result = await self.server1.stop()
+ self.assertTrue(result)
+
+ result = await self.server2.stop()
+ self.assertTrue(result)
+
+ await self.server1.start(sleep=30)
+
+ result = await self.client1.query('show status')
+ self.assertEqual(result.pop('data'), [
+ {'name': 'status', 'value': 'running | synchronizing'}])
+
+ result = await self.client0.query('drop server "localhost:9012"')
+ self.assertEqual(
+ result.pop('success_msg'),
+ "Successfully dropped server 'localhost:9012'.")
+ self.db.servers.remove(self.server2)
+
+ time.sleep(1)
+
+ for client in (self.client0, self.client1):
+ result = await client.query('list servers status')
+ self.assertEqual(result.pop('servers'), [['running'], ['running']])
+
+ await self.db.add_replica(self.server3, 1)
+ await self.assertIsRunning(self.db, self.client0, timeout=35)
+
+ self.client0.close()
+ self.client1.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestServer())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+import re
+import calendar
+import datetime
+import collections
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+# Compression OFF:
+# du --bytes testdir/dbpath0/dbtest/shards/
+# 423114 testdir/dbpath0/dbtest/shards/
+# 416266 (optimized)
+
+# Compression ON:
+# du --bytes testdir/dbpath0/dbtest/shards/
+# 222314 testdir/dbpath0/dbtest/shards/
+# 153380 (optimized)
+
+SYSLOG = '/home/joente/syslog.log'
+FMT = '%b %d %H:%M:%S'
+MTCH = re.compile(
+ r'(\w\w\w\s[\d\s]\d\s\d\d:\d\d:'
+ r'\d\d)\s(\w+)\s([\w\-\.\/@]+)(\[\d+\])?:\s(.*)')
+
+
+class TestSyslog(TestBase):
+ title = 'Test with syslog data'
+
+ async def insert_syslog(self, batch_size=100):
+
+ with open(SYSLOG, 'r') as f:
+ lines = f.readlines()
+
+ points = collections.defaultdict(list)
+ n = 0
+
+ for line in lines:
+ r = MTCH.match(line.strip())
+ if not r:
+ continue
+
+ rtime, host, process, pid, logline = r.groups()
+ dt = datetime.datetime.strptime(rtime, FMT)
+ dt = dt.replace(year=datetime.datetime.now().year)
+ ts = calendar.timegm(dt.timetuple())
+ points['{}|{}'.format(host, process)].append([ts, logline])
+ n += 1
+ if n % batch_size == 0:
+ await self.client0.insert(points)
+ points.clear()
+
+ if points:
+ await self.client0.insert(points)
+
+ @default_test_setup(2, compression=True, duration_log='1w')
+ async def run(self):
+ await self.client0.connect()
+
+ # await self.db.add_pool(self.server1, sleep=30)
+
+ await self.db.add_replica(self.server1, 0, sleep=30)
+
+ await self.insert_syslog()
+
+ await self.client0.query('select * from /.*vbox.*/ merge as "t"')
+
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestSyslog())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+import math
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+DATA = {
+ 'series-001 float': [
+ [1471254705, 1.5],
+ [1471254707, -3.5],
+ [1471254710, -7.3]],
+ 'series-001 integer': [
+ [1471254705, 5],
+ [1471254708, -3],
+ [1471254710, -7]],
+ 'series-002 float': [
+ [1471254705, 3.5],
+ [1471254707, -2.5],
+ [1471254710, -8.3]],
+ 'series-002 integer': [
+ [1471254705, 4],
+ [1471254708, -1],
+ [1471254710, -8]],
+ 'aggr': [
+ [1447249033, 531], [1447249337, 534],
+ [1447249633, 535], [1447249937, 531],
+ [1447250249, 532], [1447250549, 537],
+ [1447250868, 530], [1447251168, 520],
+ [1447251449, 54], [1447251749, 54],
+ [1447252049, 513], [1447252349, 537],
+ [1447252649, 528], [1447252968, 531],
+ [1447253244, 533], [1447253549, 538],
+ [1447253849, 534], [1447254149, 532],
+ [1447254449, 533], [1447254748, 537]],
+ 'huge': [
+ [1471254705, 9223372036854775807],
+ [1471254706, 9223372036854775806],
+ [1471254707, 9223372036854775805],
+ [1471254708, 9223372036854775804]],
+ 'equal ts': [
+ [1471254705, 0], [1471254705, 1], [1471254705, 1],
+ [1471254707, 0], [1471254707, 1], [1471254708, 0],
+ ],
+ 'variance': [
+ [1471254705, 2.75], [1471254706, 1.75], [1471254707, 1.25],
+ [1471254708, 0.25], [1471254709, 0.5], [1471254710, 1.25],
+ [1471254711, 3.5]
+ ],
+ 'pvariance': [
+ [1471254705, 0.0], [1471254706, 0.25], [1471254707, 0.25],
+ [1471254708, 1.25], [1471254709, 1.5], [1471254710, 1.75],
+ [1471254711, 2.75], [1471254712, 3.25]
+ ],
+ 'filter': [
+ [1471254705, 5],
+ [1471254710, -3],
+ [1471254715, -7],
+ [1471254720, 7]
+ ],
+ 'one': [
+ [1471254710, 1]
+ ],
+ 'log': [
+ [1471254710, 'log line one'],
+ [1471254712, 'log line two'],
+ [1471254714, 'another line (three)'],
+ [1471254716, 'and yet one more'],
+ ],
+ 'special': [
+ [1471254705, 0.1],
+ [1471254706, math.nan],
+ [1471254707, math.inf],
+ [1471254708, -math.inf],
+ ]
+}
+
+
+class TestTags(TestBase):
+ title = 'Test tag and untag series'
+
+ @default_test_setup(3, time_precision='s')
+ async def run(self):
+ await self.client0.connect()
+
+ await self.client0.insert(DATA)
+
+ res = await self.client0.query('''
+ alter series /series.*/ tag `SERIES`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully tagged 4 series."})
+
+ res = await self.client0.query('''
+ alter series /.*/ tag `ALL`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully tagged 13 series."})
+
+ res = await self.client0.query('''
+ alter series /empty/ tag `EMPTY`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully tagged 0 series."})
+
+ await asyncio.sleep(3.0)
+
+ res = await self.client0.query('''
+ alter series `ALL` - `SERIES` tag `OTHER`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully tagged 9 series."})
+
+ await self.db.add_pool(self.server1)
+
+ await asyncio.sleep(3.0)
+
+ res = await self.client0.query('''
+ alter series /series-00(1|2) integer/ tag `SERIES_INT`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully tagged 2 series."})
+
+ res = await self.client0.query('''
+ alter series 'one' untag `OTHER`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully untagged 1 series."})
+
+ await self.assertIsRunning(self.db, self.client0, timeout=45)
+
+ await asyncio.sleep(5)
+
+ await self.db.add_replica(self.server2, 0)
+ await asyncio.sleep(3.0)
+
+ res = await self.client0.query('''
+ alter series /series-00(1|2) float/ tag `SERIES_FLOAT`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully tagged 2 series."})
+
+ res = await self.client0.query('''
+ alter series 'huge' untag `OTHER`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully untagged 1 series."})
+
+ await self.assertIsRunning(self.db, self.client0, timeout=45)
+
+ res = await self.client0.query('''
+ alter series 'one', 'huge', 'log' tag `SPECIAL`
+ ''')
+ self.assertEqual(
+ res, {"success_msg": "Successfully tagged 3 series."})
+
+ await self.client0.query('''
+ alter series 'variance', 'pvariance' untag `OTHER`
+ ''')
+
+ await self.client0.query('''
+ alter series `ALL` where type == float tag `F`
+ ''')
+
+ await self.client0.query('''
+ alter series `ALL` tag `I`
+ ''')
+
+ await asyncio.sleep(3.0)
+
+ await self.client0.query('''
+ alter series `ALL` where type != integer untag `I`
+ ''')
+
+ await self.client1.connect()
+ await self.client2.connect()
+
+ for client in (self.client0, self.client1, self.client2):
+ res = await self.client0.query('''
+ list tags name, series
+ ''')
+ tags = sorted(res['tags'])
+ self.assertEqual(tags, [
+ ["ALL", 13],
+ ["EMPTY", 0],
+ ["F", 5],
+ ["I", 7],
+ ["OTHER", 5],
+ ["SERIES", 4],
+ ["SERIES_FLOAT", 2],
+ ["SERIES_INT", 2],
+ ["SPECIAL", 3],
+ ])
+
+ for series in ('huge', 'log', 'series-001 integer', 'one'):
+ await self.client0.query('''
+ drop series '{0}'
+ '''.format(series))
+
+ await asyncio.sleep(6.0) # groups and tags are updates each 3 seconds
+
+ for client in (self.client0, self.client1, self.client2):
+ res = await self.client0.query('''
+ list tags name, series
+ ''')
+ tags = sorted(res['tags'])
+ self.assertEqual(tags, [
+ ["ALL", 9],
+ ["EMPTY", 0],
+ ["F", 5],
+ ["I", 4],
+ ["OTHER", 4],
+ ["SERIES", 3],
+ ["SERIES_FLOAT", 2],
+ ["SERIES_INT", 1],
+ ["SPECIAL", 0],
+ ])
+
+ for tag in (
+ 'ALL',
+ 'EMPTY',
+ 'OTHER',
+ 'SERIES',
+ 'SERIES_FLOAT',
+ 'SERIES_INT',
+ 'SPECIAL'):
+ await self.client0.query('''
+ drop tag `{0}`
+ '''.format(tag))
+
+ await asyncio.sleep(1.5)
+
+ for client in (self.client0, self.client1, self.client2):
+ res = await client.query('''
+ list tags name, series
+ ''')
+ tags = sorted(res['tags'])
+ self.assertEqual(tags, [
+ ["F", 5],
+ ["I", 4],
+ ])
+
+ await self.client0.query('''
+ alter tag `F` set name 'Float'
+ ''')
+
+ await asyncio.sleep(1.5)
+
+ for client in (self.client0, self.client1, self.client2):
+ res = await client.query('''
+ list tags name, series
+ ''')
+ tags = sorted(res['tags'])
+ self.assertEqual(tags, [
+ ["Float", 5],
+ ["I", 4],
+ ])
+
+ self.client2.close()
+ self.client1.close()
+ self.client0.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestTags())
--- /dev/null
+import os
+import asyncio
+import functools
+import random
+import time
+import math
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import SiriDB
+from testing import TestBase
+from testing import SiriDBAsyncUnixServer
+from testing import parse_args
+
+
+PIPE_NAME = '/tmp/dbtest_tee.sock'
+
+DATA = {
+ 'series float': [
+ [1471254705, 1.5],
+ [1471254707, -3.5],
+ [1471254710, -7.3]],
+ 'series integer': [
+ [1471254705, 5],
+ [1471254708, -3],
+ [1471254710, -7]],
+ 'aggr': [
+ [1447249033, 531], [1447249337, 534],
+ [1447249633, 535], [1447249937, 531],
+ [1447250249, 532], [1447250549, 537],
+ [1447250868, 530], [1447251168, 520],
+ [1447251449, 54], [1447251749, 54],
+ [1447252049, 513], [1447252349, 537],
+ [1447252649, 528], [1447252968, 531],
+ [1447253244, 533], [1447253549, 538],
+ [1447253849, 534], [1447254149, 532],
+ [1447254449, 533], [1447254748, 537]],
+ 'huge': [
+ [1471254705, 9223372036854775807],
+ [1471254706, 9223372036854775806],
+ [1471254707, 9223372036854775805],
+ [1471254708, 9223372036854775804]],
+ 'equal ts': [
+ [1471254705, 0], [1471254705, 1], [1471254705, 1],
+ [1471254707, 0], [1471254707, 1], [1471254708, 0],
+ ],
+ 'variance': [
+ [1471254705, 2.75], [1471254706, 1.75], [1471254707, 1.25],
+ [1471254708, 0.25], [1471254709, 0.5], [1471254710, 1.25],
+ [1471254711, 3.5]
+ ],
+ 'pvariance': [
+ [1471254705, 0.0], [1471254706, 0.25], [1471254707, 0.25],
+ [1471254708, 1.25], [1471254709, 1.5], [1471254710, 1.75],
+ [1471254711, 2.75], [1471254712, 3.25]
+ ],
+ 'filter': [
+ [1471254705, 5],
+ [1471254710, -3],
+ [1471254715, -7],
+ [1471254720, 7]
+ ],
+ 'one': [
+ [1471254710, 1]
+ ],
+ 'log': [
+ [1471254710, 'log line one'],
+ [1471254712, 'log line two'],
+ [1471254714, 'another line (three)'],
+ [1471254716, 'and yet one more'],
+ ],
+ # 'special': [
+ # [1471254705, 0.1],
+ # [1471254706, math.nan],
+ # [1471254707, math.inf],
+ # [1471254708, -math.inf],
+ # ]
+}
+
+if os.path.exists(PIPE_NAME):
+ os.unlink(PIPE_NAME)
+
+
+class TestTee(TestBase):
+ title = 'Test tee'
+
+ def on_data(self, data):
+ for k, v in data.items():
+ if k not in self._tee_data:
+ self._tee_data[k] = []
+ self._tee_data[k].extend(v)
+
+ @default_test_setup(2)
+ async def run(self):
+ self._tee_data = {}
+
+ server = SiriDBAsyncUnixServer(PIPE_NAME, self.on_data)
+
+ await server.create()
+
+ await self.client0.connect()
+
+ await asyncio.sleep(1)
+
+ await self.client0.query(
+ 'alter servers set tee_pipe_name "{}"'.format(PIPE_NAME))
+
+ await asyncio.sleep(1)
+
+ self.assertEqual(
+ await self.client0.insert(DATA),
+ {'success_msg': 'Successfully inserted 60 point(s).'})
+
+ self.assertAlmostEqual(
+ await self.client0.query('select * from "series float"'),
+ {'series float': DATA['series float']})
+
+ self.assertEqual(
+ await self.client0.query('select * from "series integer"'),
+ {'series integer': DATA['series integer']})
+
+ self.assertEqual(
+ await self.client0.query('select * from "log"'),
+ {'log': DATA['log']})
+
+ await asyncio.sleep(1)
+
+ self.assertEqual(DATA, self._tee_data)
+
+ await self.db.add_pool(self.server1, sleep=3)
+ await self.assertIsRunning(self.db, self.client0, timeout=50)
+
+ await self.client1.connect()
+
+ self._tee_data = {}
+ await self.client0.query('drop series set ignore_threshold true')
+
+ await asyncio.sleep(1)
+
+ await self.client0.query(
+ 'alter servers set tee_pipe_name "{}"'.format(PIPE_NAME))
+
+ await asyncio.sleep(1)
+
+ self.assertEqual(
+ await self.client0.insert(DATA),
+ {'success_msg': 'Successfully inserted 60 point(s).'})
+
+ await asyncio.sleep(1)
+
+ self.assertEqual(DATA, self._tee_data)
+
+ self.client0.close()
+ self.client1.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestTee())
--- /dev/null
+import asyncio
+import functools
+import random
+import time
+from testing import Client
+from testing import default_test_setup
+from testing import gen_data
+from testing import gen_points
+from testing import gen_series
+from testing import InsertError
+from testing import PoolError
+from testing import QueryError
+from testing import run_test
+from testing import Series
+from testing import Server
+from testing import ServerError
+from testing import SiriDB
+from testing import TestBase
+from testing import UserAuthError
+from testing import parse_args
+
+
+class TestUser(TestBase):
+ title = 'Test user object'
+
+ @default_test_setup(3)
+ async def run(self):
+ a = await self.client0.connect()
+
+ result = await self.client0.query('list users ')
+ self.assertEqual(result.pop('users'), [['iris', 'full']])
+
+ with self.assertRaises(QueryError):
+ await self.client0.query('create user "sasientje" ')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'User name should be at least 2 characters.'):
+ await self.client0.query('create user "s" set password "123456" ')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'User name contains illegal characters.*'):
+ await self.client0.query('create user " " set password "123456" ')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ 'Password should be at least 4 characters.'):
+ await self.client0.query('create user "aa" set password "123" ')
+
+ result = await self.client0.query(
+ 'create user "sasientje" set password "blabla" ')
+ self.assertEqual(
+ result.pop('success_msg'),
+ "Successfully created user 'sasientje'.")
+
+ result = await self.client0.query('list users where access < modify ')
+ self.assertEqual(result.pop('users'), [['sasientje', 'no access']])
+
+ result = await self.client0.query('grant modify to user "sasientje" ')
+ self.assertEqual(
+ result.pop('success_msg'),
+ "Successfully granted permissions to user 'sasientje'.")
+
+ await self.db.add_replica(self.server1, 0)
+ await self.assertIsRunning(self.db, self.client0, timeout=20)
+ await asyncio.sleep(5)
+ await self.db.add_pool(self.server2)
+
+ await self.client1.connect()
+ await self.client2.connect()
+
+ await asyncio.sleep(20)
+
+ result = await self.client1.query('list users where access < full ')
+ self.assertEqual(result.pop('users'), [['sasientje', 'modify']])
+
+ result = await self.client1.query(
+ 'revoke write from user "sasientje" ')
+ self.assertEqual(
+ result.pop('success_msg'),
+ "Successfully revoked permissions from user 'sasientje'.")
+
+ result = await self.client1.query(
+ 'grant show, count to user "sasientje"')
+ result = await self.client1.query(
+ 'list users where access < modify ')
+ self.assertEqual(
+ result.pop('users'),
+ [['sasientje', 'alter, count, drop and show']])
+
+ result = await self.client1.query(
+ 'create user "pee" set password "hihihaha" ')
+ time.sleep(0.1)
+ result = await self.client0.query('list users where name ~ "p"')
+ self.assertEqual(result.pop('users'), [['pee', 'no access']])
+
+ self.client0.close()
+ result = await self.server0.stop()
+ self.assertTrue(result)
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "Password should be at least 4 characters."):
+ result = await self.client1.query(
+ 'alter user "sasientje" set password "dag" ')
+
+ result = await self.client1.query(
+ 'alter user "sasientje" set password "dagdag"')
+
+ await self.server0.start(sleep=35)
+
+ self.client0 = Client(
+ self.db,
+ self.server0,
+ username="sasientje",
+ password="dagdag")
+
+ await self.client0.connect()
+ result = await self.client0.query("show who_am_i ")
+ self.assertEqual(result['data'][0]['value'], 'sasientje')
+
+ with self.assertRaisesRegex(
+ UserAuthError,
+ "Access denied. User 'sasientje' has no 'insert' privileges."):
+ result = await self.client0.insert({'no access test': [[1, 1.0]]})
+
+ result = await self.client1.query('drop user "sasientje" ')
+ self.assertEqual(
+ result.pop('success_msg'),
+ "Successfully dropped user 'sasientje'.")
+ time.sleep(0.1)
+
+ for client in (self.client0, self.client1, self.client2):
+ result = await client.query('count users')
+ self.assertEqual(
+ result.pop('users'), 2, msg='Expecting 2 users (iris and pee)')
+
+ result = await self.client0.query('count users where name == "pee"')
+ self.assertEqual(result.pop('users'), 1, msg='Expecting 1 user (pee)')
+
+ with self.assertRaisesRegex(
+ UserAuthError,
+ "Access denied. User 'sasientje' has no 'grant' privileges."):
+ result = await self.client0.query('grant full to user "pee" ')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "User name should be at least 2 characters."):
+ result = await self.client1.query('alter user "pee" set name "p" ')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "^User name contains illegal characters.*"):
+ result = await self.client1.query(
+ 'alter user "pee" set name " p " ')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "User 'iris' already exists."):
+ result = await self.client1.query(
+ 'alter user "pee" set name "iris" ')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "User 'iris' already exists."):
+ result = await self.client1.query(
+ 'alter user "pee" set name "iris" ')
+
+ with self.assertRaisesRegex(
+ QueryError,
+ "Cannot find user: 'Pee'"):
+ result = await self.client1.query(
+ 'alter user "Pee" set name "PPP" ')
+
+ result = await self.client1.query('alter user "pee" set name "Pee"')
+ self.assertEqual(
+ result.pop('success_msg'), "Successfully updated user 'Pee'.")
+
+ time.sleep(0.1)
+ result = await self.client2.query('list users where name == "Pee" ')
+ self.assertEqual(result.pop('users'), [['Pee', 'no access']])
+
+ self.client0.close()
+ self.client1.close()
+ self.client2.close()
+
+
+if __name__ == '__main__':
+ parse_args()
+ run_test(TestUser())
--- /dev/null
+import asyncio
+import logging
+from .client import Client
+from .client import InsertError
+from .client import PoolError
+from .client import QueryError
+from .client import ServerError
+from .client import UserAuthError
+from .helpers import cleanup
+from .helpers import gen_data
+from .helpers import gen_points
+from .helpers import gen_series
+from .server import Server
+from .siridb import SiriDB
+from .testbase import default_test_setup
+from .testbase import TestBase
+from .series import Series
+from .pipe_client import PipeClient as SiriDBAsyncUnixConnection
+from .pipe_server import PipeServer as SiriDBAsyncUnixServer
+from .args import parse_args
+from .task import Task
+
+
+async def _run_test(test, loglevel):
+ logger = logging.getLogger()
+ logger.setLevel(loglevel)
+ task = Task(test.title)
+
+ try:
+ await test.run()
+ except Exception as e:
+ task.stop(success=False)
+ raise e
+ else:
+ task.stop(success=True)
+
+ logger.setLevel('CRITICAL')
+
+ await task.task
+
+
+def run_test(test, loglevel='CRITICAL'):
+ assert isinstance(test, TestBase)
+ loop = asyncio.get_event_loop()
+ cleanup()
+ loop.run_until_complete(_run_test(test, loglevel))
--- /dev/null
+import os
+import subprocess
+import argparse
+from .server import Server
+from .color import Color
+from .constants import SIRIDBC
+
+
+def is_valgrind_installed():
+ with open(os.devnull, 'w') as fnull:
+ try:
+ subprocess.call(['valgrind'], stdout=fnull, stderr=fnull)
+ except OSError as e:
+ if e.errno == os.errno.ENOENT:
+ return False
+ return True
+
+
+def print_siridb_version(args):
+ fn = SIRIDBC.format(BUILDTYPE=args.build)
+ try:
+ p = subprocess.Popen(
+ [fn, '--version'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ except FileNotFoundError:
+ print(Color.error(f'Cannot find: {fn}'))
+ exit(1)
+
+ output, err = p.communicate()
+ rc = p.returncode
+ if rc or err:
+ print(Color.error(f'Cannot use: {fn}'))
+ exit(1)
+
+ print(f'Test with: {Color.info(output.decode().splitlines()[0])}')
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ '-t', '--terminal',
+ choices=['xterm', 'xfce4-terminal'],
+ default=None,
+ help='Start SiriDB servers in a terminal. If no terminal is given '
+ 'process put their output in log files.')
+
+ parser.add_argument(
+ '-m', '--mem-check',
+ action='store_true',
+ help='Use `valgrind` for memory errors and leaks.')
+
+ parser.add_argument(
+ '-k', '--keep',
+ action='store_true',
+ help='Only valid when a terminal is used. This will keep the terminal '
+ 'open.')
+
+ parser.add_argument(
+ '-b', '--build',
+ choices=['Release', 'Debug'],
+ default='Release',
+ help='Choose either the Release or Debug build.')
+
+ parser.add_argument(
+ '-l', '--log-level',
+ default='critical',
+ help='set the log level',
+ choices=['debug', 'info', 'warning', 'error', 'critical'])
+
+ args = parser.parse_args()
+
+ has_valgrind = is_valgrind_installed()
+
+ print_siridb_version(args)
+ print("Test using valgrind for memory errors and leaks: ", end='')
+ if args.mem_check and not has_valgrind:
+ print(Color.warning('disabled (!! valgrind not found !!)'))
+ elif not args.mem_check:
+ print(Color.warning('disabled'))
+ else:
+ print(Color.success('enabled'))
+
+ Server.MEM_CHECK = args.mem_check and has_valgrind
+ Server.HOLD_TERM = args.keep
+ Server.TERMINAL = args.terminal
+ Server.BUILDTYPE = args.build
+ Server.LOG_LEVEL = args.log_level.upper()
--- /dev/null
+import sys
+import asyncio
+import functools
+import logging
+import random
+import time
+from siridb.connector import SiriDBClient
+from siridb.connector.lib.exceptions import AuthenticationError
+from siridb.connector.lib.exceptions import InsertError
+from siridb.connector.lib.exceptions import PoolError
+from siridb.connector.lib.exceptions import QueryError
+from siridb.connector.lib.exceptions import ServerError
+from siridb.connector.lib.exceptions import UserAuthError
+from .helpers import gen_points
+from .server import Server
+
+
+class Client:
+ def __init__(self, db, servers, username='iris', password='siri'):
+ self.db = db
+
+ if isinstance(servers, Server):
+ servers = [servers]
+
+ self.cluster = SiriDBClient(
+ username=username,
+ password=password,
+ dbname=db.dbname,
+ hostlist=[
+ (server.server_address, server.listen_client_port)
+ for server in servers
+ ])
+
+ self.query = self.cluster.query
+ self.insert = self.cluster.insert
+
+ async def connect(self):
+ logging.info('Create client connection to database {}'.format(
+ self.db.dbname))
+ await self.cluster.connect()
+
+ def close(self):
+ logging.info('Closing connection to database {}'.format(
+ self.db.dbname))
+ self.cluster.close()
+
+ async def insert_some_series(self,
+ series,
+ n=0.01,
+ timeout=None,
+ points=functools.partial(gen_points, n=1)):
+ random.shuffle(series)
+
+ n = int(len(series) * n)
+
+ assert (n <= len(series) and n > 0)
+
+ data = {s.name: s.add_points(points()) for s in series[:n]}
+
+ while True:
+ try:
+ await self.insert(data)
+ except PoolError as e:
+ if not timeout:
+ raise e
+ timeout -= 1
+ await asyncio.sleep(1.0)
+ else:
+ break
+
+ if timeout is not None:
+ time.sleep(0.1)
+
+ for s in series[:n]:
+ s.commit_points()
--- /dev/null
+
+NORMAL = '\x1B[0m'
+RED = '\x1B[31m'
+GREEN = '\x1B[32m'
+YELLOW = '\x1B[33m'
+LYELLOW = '\x1b[93m'
+
+
+class Color:
+
+ @staticmethod
+ def success(text):
+ return f'{GREEN}{text}{NORMAL}'
+
+ @staticmethod
+ def warning(text):
+ return f'{YELLOW}{text}{NORMAL}'
+
+ @staticmethod
+ def error(text):
+ return f'{RED}{text}{NORMAL}'
+
+ @staticmethod
+ def info(text):
+ return f'{LYELLOW}{text}{NORMAL}'
--- /dev/null
+TEST_DIR = './testdir'
+PYGRAMMAR_PATH = '../grammar'
+SIRIDBC = '../{BUILDTYPE}/siridb-server'
+SERVICE = '/usr/local/bin/siridb-admin'
+VALGRIND = 'valgrind' \
+ ' --tool=memcheck' \
+ ' --error-exitcode=1' \
+ ' --leak-check=full' \
+ ' --show-leak-kinds=all' \
+ ' --track-origins=yes' \
+ ' -v '
+VALGRIND = 'valgrind' \
+ ' --tool=memcheck' \
+ ' --error-exitcode=1' \
+ ' --leak-check=full '
+
+MAX_OPEN_FILES = 512 # Default value is 32768 but with valgrind 512 is max
--- /dev/null
+import os
+import logging
+import shutil
+import time
+import random
+import functools
+import string
+from .constants import TEST_DIR
+from .series import Series
+
+_MAP_TS = {
+ 's': 10**0,
+ 'ms': 10**3,
+ 'us': 10**6,
+ 'ns': 10**9
+}
+
+
+def cleanup():
+ logging.info('Remove test dir')
+ try:
+ shutil.rmtree(os.path.join(TEST_DIR))
+ except FileNotFoundError:
+ pass
+ os.mkdir(TEST_DIR)
+
+ logging.info('Force kill all open siridb-server processes')
+ os.system('pkill -9 siridb-server')
+
+ logging.info('Force kill all open memcheck-amd64- processes')
+ os.system('pkill -9 memcheck-amd64-')
+
+
+def random_value(tp=float, mi=-100, ma=100):
+ i = random.randrange(mi, ma)
+ if tp == float:
+ return i * random.random()
+ elif tp == int:
+ return i
+
+
+def random_series_name(size=12, chars=string.ascii_letters + string.digits):
+ return ''.join(random.choice(chars) for _ in range(size))
+
+
+def gen_points(
+ n=100,
+ time_precision='s',
+ tp=float,
+ mi=-100,
+ ma=100,
+ ts_gap=1,
+ shuffle=True):
+ if isinstance(ts_gap, str):
+ if ts_gap.endswith('s'):
+ ts_gap = int(ts_gap[:-1]) * _MAP_TS[time_precision]
+ elif ts_gap.endswith('m'):
+ ts_gap = int(ts_gap[:-1]) * _MAP_TS[time_precision] * 60
+ elif ts_gap.endswith('h'):
+ ts_gap = int(ts_gap[:-1]) * _MAP_TS[time_precision] * 3600
+ elif ts_gap.endswith('d'):
+ ts_gap = int(ts_gap[:-1]) * _MAP_TS[time_precision] * 3600 * 24
+ elif ts_gap.endswith('w'):
+ ts_gap = int(ts_gap[:-1]) * _MAP_TS[time_precision] * 3600 * 24 * 7
+
+ end = int(time.time() * _MAP_TS[time_precision])
+ start = end - (n * ts_gap)
+ timestamps = list(range(start, end, ts_gap))
+ if shuffle:
+ random.shuffle(timestamps)
+ return [[ts, random_value(tp, mi, ma)] for ts in timestamps]
+
+
+def gen_data(points=functools.partial(gen_points), n=100):
+ return {random_series_name(): points() for _ in range(n)}
+
+
+def gen_series(n=10000):
+ return [Series(random_series_name()) for _ in range(n)]
--- /dev/null
+import logging
+import asyncio
+from siridb.connector import SiriDBProtocol
+from siridb.connector.lib.connection import SiriDBAsyncConnection
+
+
+class PipeClient(SiriDBAsyncConnection):
+ def __init__(self, pipe_name, loop=None):
+ self._pipe_name = pipe_name
+ self._protocol = None
+
+ async def connect(self, username, password, dbname, loop=None):
+ loop = loop or asyncio.get_event_loop()
+
+ transport, self._protocol = await loop.create_unix_connection(
+ path=self._pipe_name,
+ protocol_factory=lambda: SiriDBProtocol(
+ username, password, dbname))
+
+ try:
+ res = await self._protocol.auth_future
+ except Exception as exc:
+ logging.debug('Authentication failed: {}'.format(exc))
+ transport.close()
+ raise exc
+ else:
+ self._protocol.on_authenticated()
--- /dev/null
+
+import logging
+import asyncio
+import struct
+import qpack
+from siridb.connector import SiriDBProtocol
+from siridb.connector.lib.connection import SiriDBAsyncConnection
+
+
+class Package:
+
+ __slots__ = ('pid', 'length', 'tipe', 'checkbit', 'data')
+
+ struct_datapackage = struct.Struct('<IHBB')
+
+ def __init__(self, barray):
+ self.length, self.pid, self.tipe, self.checkbit = \
+ self.__class__.struct_datapackage.unpack_from(barray, offset=0)
+ self.length += self.__class__.struct_datapackage.size
+ self.data = None
+
+ def extract_data_from(self, barray):
+ try:
+ self.data = qpack.unpackb(
+ barray[self.__class__.struct_datapackage.size:self.length],
+ decode='utf-8')
+ finally:
+ del barray[:self.length]
+
+
+class SiriDBServerProtocol(asyncio.Protocol):
+
+ def __init__(self, on_package_received):
+ self._buffered_data = bytearray()
+ self._data_package = None
+ self._on_package_received = on_package_received
+
+ def data_received(self, data):
+ '''
+ override asyncio.Protocol
+ '''
+ self._buffered_data.extend(data)
+ while self._buffered_data:
+ size = len(self._buffered_data)
+ if self._data_package is None:
+ if size < Package.struct_datapackage.size:
+ return None
+ self._data_package = Package(self._buffered_data)
+ if size < self._data_package.length:
+ return None
+ try:
+ self._data_package.extract_data_from(self._buffered_data)
+ except KeyError as e:
+ logging.error('Unsupported package received: {}'.format(e))
+ except Exception as e:
+ logging.exception(e)
+ # empty the byte-array to recover from this error
+ self._buffered_data.clear()
+ else:
+ self._on_package_received(self._data_package.data)
+ self._data_package = None
+
+
+class PipeServer(SiriDBAsyncConnection):
+ def __init__(self, pipe_name, on_data):
+ self._pipe_name = pipe_name
+ self._protocol = None
+ self._server = None
+ self._on_data_cb = on_data
+
+ async def create(self, loop=None):
+ loop = loop or asyncio.get_event_loop()
+
+ self._server = await loop.create_unix_server(
+ path=self._pipe_name,
+ protocol_factory=lambda: SiriDBServerProtocol(self._on_data))
+
+ def _on_data(self, data):
+ '''
+ series names are returned as c strings (0 terminated)
+ '''
+ data = {k.rstrip('\x00'): v for k, v in data.items()}
+ self._on_data_cb(data)
--- /dev/null
+class Series:
+ def __init__(self, name):
+ self.name = name
+ self.points = []
+ self._points = None
+
+ def add_points(self, points):
+ self.commit_points()
+ self._points = points
+ return points
+
+ def commit_points(self):
+ if self._points:
+ for point in self._points:
+ self.points.append(point)
+ self._points = None
--- /dev/null
+import os
+import logging
+import configparser
+import subprocess
+import psutil
+import asyncio
+import time
+import socket
+import platform
+from .constants import SIRIDBC
+from .constants import TEST_DIR
+from .constants import VALGRIND
+from .constants import MAX_OPEN_FILES
+from .color import Color
+
+
+MEM_PROC = \
+ 'memcheck-amd64-' if platform.architecture()[0] == '64bit' else \
+ 'memcheck-x86-li'
+
+
+def get_file_content(fn):
+ with open(fn, 'r') as f:
+ data = f.read()
+ return data
+
+
+class Server:
+ HOLD_TERM = False
+ GEOMETRY = '140x60'
+ MEM_CHECK = False
+ BUILDTYPE = 'Release'
+ SERVER_ADDRESS = '%HOSTNAME'
+ IP_SUPPORT = 'ALL' # ALL, IPV4ONLY
+ TERMINAL = None # one of [ 'XTERM', 'XFCE4_TERMINAL', None ]
+ BIND_CLIENT_ADDRESS = "::"
+ BIND_SERVER_ADDRESS = "::"
+
+ def __init__(self,
+ n,
+ title,
+ optimize_interval=300,
+ heartbeat_interval=30,
+ buffer_sync_interval=500,
+ compression=True,
+ auto_duration=True,
+ pipe_name=None,
+ **unused):
+ self.n = n
+ self.test_title = title.lower().replace(' ', '_')
+ self.compression = compression
+ self.auto_duration = auto_duration
+ self.enable_pipe_support = int(bool(pipe_name))
+ self.pipe_name = \
+ 'siridb_client.sock' if not self.enable_pipe_support else \
+ pipe_name
+ self.http_status_port = 8080 + n
+ self.listen_client_port = 9000 + n
+ self.listen_backend_port = 9010 + n
+ self.http_api_port = 9020 + n
+ self.buffer_sync_interval = buffer_sync_interval
+ self._server_address = self.SERVER_ADDRESS
+ self.server_address = \
+ self._server_address.lstrip('[').rstrip(']').replace(
+ '%HOSTNAME', socket.gethostname())
+ self.ip_support = self.IP_SUPPORT
+ self.optimize_interval = optimize_interval
+ self.heartbeat_interval = heartbeat_interval
+ self.bind_client_address = self.BIND_CLIENT_ADDRESS
+ self.bind_server_address = self.BIND_SERVER_ADDRESS
+ self.cfgfile = os.path.join(TEST_DIR, 'siridb{}.conf'.format(self.n))
+ self.dbpath = os.path.join(TEST_DIR, 'dbpath{}'.format(self.n))
+ self.name = 'SiriDB:{}'.format(self.listen_backend_port)
+ self.pid = None
+ self.buffer_path = None
+ self.buffer_size = None
+
+ @property
+ def addr(self):
+ return '{}:{}'.format(self.server_address, self.listen_client_port)
+
+ def create(self):
+ logging.info('Create server {}'.format(self.name))
+
+ config = configparser.RawConfigParser()
+ config.add_section('siridb')
+ config.set('siridb', 'listen_client_port', self.listen_client_port)
+ config.set('siridb', 'bind_client_address', self.bind_client_address)
+ config.set('siridb', 'bind_server_address', self.bind_server_address)
+ config.set('siridb', 'server_name', '{}:{}'.format(
+ self._server_address,
+ self.listen_backend_port))
+ config.set('siridb', 'ip_support', self.ip_support)
+ config.set('siridb', 'optimize_interval', self.optimize_interval)
+ config.set('siridb', 'heartbeat_interval', self.heartbeat_interval)
+ config.set('siridb', 'buffer_sync_interval', self.buffer_sync_interval)
+ config.set('siridb', 'default_db_path', self.dbpath)
+ config.set('siridb', 'max_open_files', MAX_OPEN_FILES)
+ config.set('siridb', 'enable_shard_compression', int(self.compression))
+ config.set(
+ 'siridb', 'enable_shard_auto_duration',
+ int(self.auto_duration))
+ config.set('siridb', 'enable_pipe_support', self.enable_pipe_support)
+ config.set('siridb', 'pipe_client_name', self.pipe_name)
+ config.set('siridb', 'http_status_port', self.http_status_port)
+ config.set('siridb', 'http_api_port', self.http_api_port)
+
+ with open(self.cfgfile, 'w') as configfile:
+ config.write(configfile)
+
+ try:
+ os.mkdir(self.dbpath)
+ except FileExistsError:
+ pass
+
+ def _get_pid_set(self):
+ try:
+ ret = set(map(int, subprocess.check_output([
+ 'pgrep',
+ MEM_PROC if self.MEM_CHECK else 'siridb-server']).split()))
+ except subprocess.CalledProcessError:
+ ret = set()
+ return ret
+
+ async def start(self, sleep=None):
+ prev = self._get_pid_set()
+ if self.TERMINAL == 'xfce4-terminal':
+ self.proc = subprocess.Popen(
+ 'xfce4-terminal -e "{}{} --config {} --log-colorized"'
+ ' --title {} --geometry={}{}'
+ .format(VALGRIND if self.MEM_CHECK else '',
+ SIRIDBC.format(BUILDTYPE=self.BUILDTYPE),
+ self.cfgfile,
+ self.name,
+ self.GEOMETRY,
+ ' -H' if self.HOLD_TERM else ''),
+ shell=True)
+ elif self.TERMINAL == 'xterm':
+ self.proc = subprocess.Popen(
+ 'xterm {}-title {} -geometry {} -e "{}{} --config {}"'
+ .format('-hold ' if self.HOLD_TERM else '',
+ self.name,
+ self.GEOMETRY,
+ VALGRIND if self.MEM_CHECK else '',
+ SIRIDBC.format(BUILDTYPE=self.BUILDTYPE),
+ self.cfgfile),
+ shell=True)
+ elif self.TERMINAL is None:
+ errfn = f'testdir/{self.test_title}-{self.name}-err.log'
+ outfn = f'testdir/{self.test_title}-{self.name}-out.log'
+ with open(errfn, 'a') as err:
+ with open(outfn, 'a') as out:
+ self.proc = subprocess.Popen(
+ '{}{} --config {}'
+ .format(VALGRIND if self.MEM_CHECK else '',
+ SIRIDBC.format(BUILDTYPE=self.BUILDTYPE),
+ self.cfgfile),
+ stderr=err,
+ stdout=out,
+ shell=True)
+
+ await asyncio.sleep(5)
+
+ my_pid = self._get_pid_set() - prev
+ if len(my_pid) != 1:
+ if self.TERMINAL is None:
+ if os.path.exists(outfn) and os.path.getsize(outfn):
+ reasoninfo = get_file_content(outfn)
+ elif os.path.exists(errfn):
+ reasoninfo = get_file_content(errfn)
+ else:
+ reasoninfo = 'unknown'
+ assert 0, (
+ f'{Color.error("Failed to start SiriDB server")}\n'
+ f'{Color.info(reasoninfo)}\n')
+ else:
+ assert 0, (
+ 'Failed to start SiriDB server. A possible reason could '
+ 'be that another process is using the same port.')
+
+ self.pid = my_pid.pop()
+ if sleep:
+ await asyncio.sleep(sleep)
+
+ async def stop(self, timeout=20):
+ if self.is_active():
+ os.system('kill {}'.format(self.pid))
+
+ while (timeout and self.is_active()):
+ await asyncio.sleep(1.0)
+ timeout -= 1
+
+ self.proc.communicate()
+ assert (self.proc.returncode == 0)
+
+ if timeout:
+ self.pid = None
+ return True
+
+ return False
+
+ async def stopstop(self):
+ if self.is_active():
+ os.system('kill {}'.format(self.pid))
+ await asyncio.sleep(0.2)
+
+ if self.is_active():
+ os.system('kill {}'.format(self.pid))
+ await asyncio.sleep(1.0)
+
+ if self.is_active():
+ return False
+
+ self.pid = None
+ return True
+
+ def set_buffer_size(self, db, buffer_size):
+ self.buffer_size = buffer_size
+ config = configparser.RawConfigParser()
+ config.add_section('buffer')
+ if self.buffer_path is not None:
+ config.set('buffer', 'path', self.buffer_path)
+ config.set('buffer', 'size', self.buffer_size)
+ with open(os.path.join(
+ self.dbpath, db.dbname, 'database.conf'), 'w') as f:
+ config.write(f)
+
+ def set_buffer_path(self, db, buffer_path):
+ assert(buffer_path.endswith('/'))
+ curfile = os.path.join(self.dbpath, db.dbname, 'buffer.dat') \
+ if self.buffer_path is None else \
+ os.path.join(self.buffer_path, 'buffer.dat')
+ if not os.path.exists(buffer_path):
+ os.makedirs(buffer_path)
+ os.rename(curfile, os.path.join(buffer_path, 'buffer.dat'))
+ self.buffer_path = buffer_path
+ config = configparser.RawConfigParser()
+ config.add_section('buffer')
+ config.set('buffer', 'path', self.buffer_path)
+ if self.buffer_size is not None:
+ config.set('buffer', 'size', self.buffer_size)
+ with open(os.path.join(
+ self.dbpath, db.dbname, 'database.conf'), 'w') as f:
+ config.write(f)
+
+ def kill(self):
+ print("!!!!!!!!!!!! KILLL !!!!!!!!!!")
+ os.system('kill -9 {}'.format(self.pid))
+ self.pid = None
+
+ def is_active(self):
+ return False if self.pid is None else psutil.pid_exists(self.pid)
--- /dev/null
+import os
+import logging
+import random
+import asyncio
+from .constants import SERVICE
+
+VERBOSE = ' --verbose'
+
+
+class SiriDB:
+
+ LOG_LEVEL = 'CRITICAL'
+
+ def __init__(self,
+ dbname='dbtest',
+ time_precision='s',
+ buffer_path='',
+ duration_log='1d',
+ duration_num='1w',
+ buffer_size=1024,
+ **unused):
+ self.dbname = dbname
+ self.time_precision = time_precision
+ self.buffer_path = buffer_path
+ self.duration_log = duration_log
+ self.duration_num = duration_num
+ self.buffer_size = buffer_size
+ self.servers = []
+
+ async def create_on(self, server, sleep=None):
+ await asyncio.sleep(1)
+
+ logging.info('Create database {} on {}'.format(
+ self.dbname,
+ server.name))
+
+ rc = os.system(
+ '{service} '
+ '-u sa -p siri -s {addr} '
+ 'new-database '
+ '--db-name {dbname} '
+ '--time-precision {time_precision} '
+ '--duration-log {duration_log} '
+ '--duration-num {duration_num} '
+ '--buffer-size {buffer_size}'
+ '{verbose}'.format(
+ service=SERVICE,
+ addr=server.addr,
+ verbose=VERBOSE if self.LOG_LEVEL == 'DEBUG'
+ else ' >/dev/null',
+ **vars(self)))
+
+ assert rc == 0, 'Expected rc = 0 but got rc = {}'.format(rc)
+
+ self.servers.append(server)
+
+ if sleep:
+ await asyncio.sleep(sleep)
+
+ async def drop_db(
+ self,
+ server,
+ username='iris',
+ password='siri',
+ sleep=None):
+
+ rc = os.system(
+ '{service} '
+ '-u sa -p siri -s {addr} '
+ 'drop-database '
+ '--db-name {dbname} '
+ '--ignore-offline '
+ '--force{verbose}'.format(
+ service=SERVICE,
+ addr=server.addr,
+ verbose=VERBOSE if self.LOG_LEVEL == 'DEBUG'
+ else ' >/dev/null',
+ **vars(self)))
+
+ assert rc == 0, 'Expected rc = 0 but got rc = {}'.format(rc)
+
+ self.servers.append(server)
+
+ if sleep:
+ await asyncio.sleep(sleep)
+
+ async def add_replica(
+ self,
+ server,
+ pool,
+ username='iris',
+ password='siri',
+ remote_server=None,
+ sleep=None):
+
+ if remote_server is None:
+ remote_server = random.choice(self.servers)
+
+ rc = os.system(
+ '{service} '
+ '-u sa -p siri -s {addr} '
+ 'new-replica '
+ '--db-name {dbname} '
+ '--db-server {dbaddr} '
+ '--db-user {dbuser} '
+ '--db-password {dbpassword} '
+ '--pool {pool} '
+ '--force{verbose}'.format(
+ service=SERVICE,
+ addr=server.addr,
+ dbaddr=remote_server.addr,
+ dbuser=username,
+ dbpassword=password,
+ pool=pool,
+ verbose=VERBOSE if self.LOG_LEVEL == 'DEBUG'
+ else ' >/dev/null',
+ **vars(self)))
+
+ assert rc == 0, 'Expected rc = 0 but got rc = {}'.format(rc)
+
+ self.servers.append(server)
+
+ if sleep:
+ await asyncio.sleep(sleep)
+
+ async def add_pool(
+ self,
+ server,
+ username='iris',
+ password='siri',
+ remote_server=None,
+ sleep=None):
+
+ if remote_server is None:
+ remote_server = random.choice(self.servers)
+
+ rc = os.system(
+ '{service} '
+ '-u sa -p siri -s {addr} '
+ 'new-pool '
+ '--db-name {dbname} '
+ '--db-server {dbaddr} '
+ '--db-user {dbuser} '
+ '--db-password {dbpassword} '
+ '--force{verbose}'.format(
+ service=SERVICE,
+ addr=server.addr,
+ dbaddr=remote_server.addr,
+ dbuser=username,
+ dbpassword=password,
+ verbose=VERBOSE if self.LOG_LEVEL == 'DEBUG'
+ else ' >/dev/null',
+ **vars(self)))
+
+ assert rc == 0, 'Expected rc = 0 but got rc = {}'.format(rc)
+
+ self.servers.append(server)
+
+ if sleep:
+ await asyncio.sleep(sleep)
--- /dev/null
+SPINNER1 = \
+ ('▁', '▂', '▃', '▄', '▅', '▆', '▇', '█', '▇', '▆', '▅', '▄', '▃', '▁')
+SPINNER2 = \
+ ('⠁', '⠂', '⠄', '⡀', '⢀', '⠠', '⠐', '⠈')
+SPINNER3 = \
+ ('◐', '◓', '◑', '◒')
+
+
+class Spinner:
+
+ def __init__(self, charset=SPINNER3):
+ self._idx = 0
+ self._charset = charset
+ self._len = len(charset)
+
+ @property
+ def next(self):
+ char = self._charset[self._idx]
+ self._idx += 1
+ self._idx %= self._len
+ return char
--- /dev/null
+import sys
+import time
+import asyncio
+from .spinner import Spinner
+from .color import Color
+
+
+class Task:
+ def __init__(self, title):
+ self.running = True
+ self.task = asyncio.ensure_future(self.process())
+ self.success = False
+ self.title = title
+ self.start = time.time()
+
+ def stop(self, success):
+ self.running = False
+ self.success = success
+ self.duration = time.time() - self.start
+
+ async def process(self):
+ spinner = Spinner()
+ while self.running:
+ sys.stdout.write(f'{self.title:.<76}{spinner.next}\r')
+ sys.stdout.flush()
+ await asyncio.sleep(0.2)
+
+ if self.success:
+ print(
+ f'{self.title:.<76}'
+ f'{Color.success("OK")} ({self.duration:.2f} seconds)')
+ else:
+ print(
+ f'{self.title:.<76}'
+ f'{Color.error("FAILED")} ({self.duration:.2f} seconds)')
--- /dev/null
+import unittest
+import time
+import asyncio
+import random
+import math
+import collections
+from .siridb import SiriDB
+from .server import Server
+from .client import Client
+
+
+def default_test_setup(nservers=1, **kwargs):
+ def wrapper(func):
+ async def wrapped(self):
+ self.db = SiriDB(**kwargs)
+
+ self.servers = [
+ Server(n, title=self.title, **kwargs) for n in range(nservers)]
+ for n, server in enumerate(self.servers):
+ setattr(self, 'server{}'.format(n), server)
+ setattr(self, 'client{}'.format(n), Client(self.db, server))
+ server.create()
+ await server.start()
+
+ time.sleep(2.0)
+
+ await self.db.create_on(self.server0, sleep=5)
+
+ close = await func(self)
+
+ if Server.TERMINAL is None or Server.HOLD_TERM is not True:
+ for server in self.servers:
+ result = await server.stop()
+ self.assertTrue(
+ result,
+ msg='Server {} did not close correctly'.format(
+ server.name))
+
+ return wrapped
+ return wrapper
+
+
+class TestBase(unittest.TestCase):
+
+ title = 'No title set'
+
+ async def run():
+ raise NotImplementedError()
+
+ async def assertIsRunning(self, db, client, timeout=None):
+ while True:
+ result = await client.query('list servers name, status')
+ result = result['servers']
+ try:
+ assert len(result) == len(self.db.servers), \
+ 'Server(s) are missing: {} (expecting: {})'.format(
+ result, self.db.servers)
+ except AssertionError as e:
+ if not timeout:
+ raise e
+ else:
+ try:
+ assert all([
+ status == 'running' for name, status in result]), \
+ 'Not all servers have status running: {}'.format(
+ result)
+ except AssertionError as e:
+ if not timeout:
+ raise e
+ else:
+ break
+
+ if timeout is not None:
+ timeout -= 1
+
+ await asyncio.sleep(1.0)
+
+ async def assertSeries(self, client, series):
+ d = {s.name: s.points for s in series if s.points}
+
+ result = await client.query('list series name, length limit {}'.format(
+ len(series)))
+ result = {name: length for name, length in result['series']}
+ for s in series:
+ if s.points:
+ length = result.get(s.name, None)
+ assert length is not None, \
+ 'series {!r} is missing in the result'.format(s.name)
+ assert length == len(s.points) or \
+ s.commit_points() or \
+ length == len(s.points), \
+ 'expected {} point(s) but found {} point(s) ' \
+ 'for series {!r}' \
+ .format(len(s.points), length, s.name)
+
+ n = min(len(series), 10)
+ li = list(d.keys())
+ random.shuffle(li)
+ for i in range(n):
+ result = await client.query('select * from "{}"'.format(li[i]))
+ points = result[li[i]]
+ expected = sorted(d[li[i]])
+ assert len(points) == len(expected), \
+ 'incorrect number of points: {}, expected: {}'.format(
+ len(points), len(expected))
+ c = collections.Counter([p[0] for p in points])
+ for i, p in enumerate(points):
+ ts, val = p
+ if c[ts] == 1:
+ self.assertEqual(p, expected[i])
+
+ def assertAlmostEqual(self, a, b, *args, **kwargs):
+ if isinstance(b, dict):
+ for series, points in b.items():
+ assert isinstance(points, list), 'Expecting a list of points'
+ for i, point in enumerate(points):
+ assert isinstance(point, list) and len(point) == 2, \
+ 'Expecting a point to be a list of 2 items'
+ super().assertEqual(a[series][i][0], point[0])
+ if isinstance(a[series][i][1], str):
+ super().assertEqual(
+ a[series][i][1].replace(',', '.'),
+ point[1].replace(',', '.'))
+ elif math.isnan(a[series][i][1]):
+ assert math.isnan(point[1]), \
+ 'Expecting point `{}` to be `nan`, got: `{}`' \
+ .format(i, point[1])
+ else:
+ super().assertAlmostEqual(
+ a[series][i][1],
+ point[1],
+ *args,
+ **kwargs)
+ else:
+ super().assertAlmostEqual(a, b, *args, **kwargs)
--- /dev/null
+/*
+ * main.c - SiriDB.
+ *
+ * author/maintainer : Jeroen van der Heijden <jeroen@transceptor.technology>
+ * contributors : https://github.com/SiriDB/siridb-server/contributors
+ * home page : https://siridb.net
+ * copyright : 2018, Transceptor Technology
+ *
+ */
+#include <locale.h>
+#include <logger/logger.h>
+#include <siri/args/args.h>
+#include <siri/db/presuf.h>
+#include <siri/err.h>
+#include <siri/help/help.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <siri/evars.h>
+#include <siri/net/tcp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+
+int main(int argc, char * argv[])
+{
+ /* set local to LC_ALL and C to force a period over comma for float */
+ (void) setlocale(LC_ALL, "C");
+
+ /* initialize random */
+ srand(time(NULL));
+
+ /* set threadpool size to 8 (default=4) */
+ putenv("UV_THREADPOOL_SIZE=8");
+
+ /* set default timezone to UTC */
+ putenv("TZ=:UTC");
+ tzset();
+
+ /* parse arguments first since this can exit the program */
+ siri_args_parse(&siri, argc, argv);
+
+ /* setup logger, this must be done before logging the first line */
+ siri_setup_logger();
+
+ /* initialize points dictionary */
+ siridb_points_init();
+
+ /* start server */
+ log_info("Starting SiriDB Server (version: %s)", SIRIDB_VERSION);
+
+ /* initialize SiriDB mutex (used for the siridb_list) */
+ if (uv_mutex_init(&siri.siridb_mutex))
+ {
+ exit(1);
+ }
+
+ /* read siridb main application configuration */
+ siri_cfg_init(&siri);
+
+ siri_evars_parse(&siri);
+
+ if (make_database_directory())
+ {
+ exit(1);
+ }
+
+ set_max_open_files_limit();
+
+ log_debug("Shard compression: %s", siri.cfg->shard_compression ? "enabled" : "disabled");
+ log_debug("Shard auto duration: %s", siri.cfg->shard_auto_duration ? "enabled" : "disabled");
+ log_debug("Pipe support: %s", siri.cfg->pipe_support ? "enabled" : "disabled");
+ log_debug("IP support: %s", sirinet_tcp_ip_support_str(siri.cfg->ip_support));
+
+ /* start SiriDB. (this start the event loop etc.) */
+ if (siri_start() && !siri_err)
+ {
+ siri_err = ERR_STARTUP;
+ }
+
+ /* free siridb */
+ siri_free();
+
+ /* destroy SiriDB mutex */
+ uv_mutex_destroy(&siri.siridb_mutex);
+
+ /* cleanup prefix-suffix allocation */
+ siridb_presuf_cleanup();
+
+ /* cleanup help */
+ siri_help_free();
+
+ log_info("Bye! (%d)\n", siri_err);
+
+ return siri_err;
+}
--- /dev/null
+.PHONY: install
+install:
+ @mkdir -p /etc/siridb/
+ @mkdir -p /var/lib/siridb/
+ @cp -n ../siridb.conf /etc/siridb/siridb.conf
+ @cp siridb-server $(INSTALL_PATH)/bin/siridb-server
+
+
+.PHONY: uninstall
+uninstall:
+ @rm -f $(INSTALL_PATH)/bin/siridb-server
--- /dev/null
+# Welcome to the SiriDB configuration file
+
+[siridb]
+#
+# SiriDB will use this address:port for it's back-end connections.
+# This must be an address that other servers can use to connect to.
+# For example IPv4, IPv6 or a fqdn are all possible. When using IPv6 be sure
+# to wrap the ip address with square brackets. For example [::1]:9010
+# The default value is %HOSTNAME:9010. The variable %HOSTNAME will be translate
+# to the systems host name.
+#
+server_name = %HOSTNAME:9010
+
+#
+# Listen for SiriDB-server connections only on localhost.
+# Use value 0.0.0.0 (or :: for IPv6) to bind to all interfaces.
+#
+bind_server_address = 127.0.0.1
+
+#
+# SiriDB will listen for client connections on this port number.
+#
+listen_client_port = 9000
+
+#
+# Listen for client connections only on localhost.
+# Use value 0.0.0.0 (or :: for IPv6) to bind to all interfaces.
+#
+bind_client_address = 127.0.0.1
+
+#
+# When ip_support is set to ALL, SiriDB will listen and connect to both IPv4
+# and IPv6 addresses.
+# Valid options are ALL, IPV4ONLY and IPV6ONLY.
+#
+ip_support = ALL
+
+#
+# SiriDB will load databases from, and create databases in this location.
+#
+db_path = /var/lib/siridb
+
+#
+# SiriDB will run an optimize task each X seconds. A value of 0 (zero) disables
+# optimizing.
+#
+optimize_interval = 3600
+
+#
+# SiriDB uses a heart-beat interval to keep connections with other servers
+# online.
+#
+heartbeat_interval = 30
+
+#
+# SiriDB can run fsync on the buffer file on an interval in milliseconds.
+# This value is set to 0 by default which tells SiriDB to run fsync after
+# each insert request. When having many insert requests per second, it can be
+# useful to use an interval like 500 milliseconds.
+#
+#buffer_sync_interval = 500
+buffer_sync_interval = 0
+
+#
+# SiriDB will not open more shard files than max_open_files. Note that the
+# total number of open files can be sligtly higher since SiriDB also needs
+# a few other files to write to.
+#
+max_open_files = 32768
+
+#
+# Use shard compression for storing data points.
+# Set value 0 to disable shard compression.
+#
+enable_shard_compression = 1
+
+#
+# Let SiriDB control shard duration when possible. When enabled, the configured
+# shard duration for both number and log values will still be used when SiriDB
+# is not able to detect a sensible duration.
+#
+enable_shard_auto_duration = 1
+
+#
+# Enable named pipe support for client connections.
+#
+enable_pipe_support = 0
+
+#
+# SiriDB will bind the client named pipe in this location.
+#
+pipe_client_name = siridb_client.sock
+
+#
+# When the HTTP status port is not set (or 0), the service will not start.
+# Otherwise the HTTP requests `/status`, `/ready` and `/healthy` are available
+# which can be used for readiness and liveness requests.
+#
+# Example usage using wget:
+#
+# wget -q -O - http://siridb-server.local:8080/status
+#
+#http_status_port = 8080
+http_status_port = 0
+
+#
+# When the HTTP API port is not set (or 0), the API service will not start.
+# Otherwise the HTTP POST requests can be user to insert or query data points.
+#
+#http_api_port = 9020
+http_api_port = 0
\ No newline at end of file
--- /dev/null
+/*
+ * argparse.c - Module for parsing command line arguments.
+ */
+#include <argparse/argparse.h>
+#include <libgen.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <xstr/xstr.h>
+#include <string.h>
+#include <assert.h>
+
+#define HELP_WIDTH 80 /* try to fit help within this screen width */
+#define ARGPARSE_ERR_SIZE 1024 /* buffer size for building err msg */
+#define ARGPARSE_HELP_SIZE 512 /* buffer size for building help */
+
+/* static function definitions */
+static void print_usage(argparse_parser_t * parser, const char * bname);
+static void print_help(argparse_parser_t * parser, const char * bname);
+static void print_error(
+ argparse_parser_t * parser,
+ char * err_msg,
+ const char * bname);
+static int process_arg(
+ argparse_parser_t * parser,
+ argparse_args_t ** arg,
+ int * argn,
+ int argc,
+ char *argv[]);
+static uint16_t get_argument(
+ argparse_args_t ** target,
+ argparse_parser_t * parser,
+ char * argument);
+static bool match_choice(char * choices, char * choice);
+static void fill_defaults(argparse_parser_t * parser);
+static void quit(argparse_parser_t * parser, int rc);
+
+
+void argparse_init(argparse_parser_t * parser)
+{
+ /* set initial show help to false */
+ parser->show_help = false;
+
+ /* create help argument */
+ parser->help.name = "help";
+ parser->help.shortcut = 'h';
+ parser->help.help = "show this help message and exit";
+ parser->help.action = ARGPARSE_STORE_TRUE;
+ parser->help.pt_value_int32_t = &parser->show_help;
+
+ /* add help argument to parser */
+ parser->args = malloc(sizeof(argparse_args_t));
+ parser->args->argument = &parser->help;
+ parser->args->next = NULL;
+}
+
+void argparse_free(argparse_parser_t * parser)
+{
+ argparse_args_t * current = parser->args;
+ argparse_args_t * next;
+
+ while (current != NULL)
+ {
+ next = current->next;
+ free(current);
+ current = next;
+ }
+}
+
+void argparse_add_argument(
+ argparse_parser_t * parser,
+ argparse_argument_t * argument)
+{
+ argparse_args_t * current = parser->args;
+
+ assert(current != NULL);
+
+ while (current->next != NULL)
+ {
+ current = current->next;
+ }
+ current->next = malloc(sizeof(argparse_args_t));
+ current->next->argument = argument;
+ current->next->next = NULL;
+}
+
+void argparse_parse(argparse_parser_t *parser, int argc, char *argv[])
+{
+ int argn;
+ int rc;
+ char buffer[ARGPARSE_ERR_SIZE];
+ const char * bname = basename(argv[0]);
+ argparse_args_t * current = NULL;
+ for (argn = 1; argn < argc; argn++)
+ {
+ rc = process_arg(parser, ¤t, &argn, argc, argv);
+ switch (rc)
+ {
+ case ARGPARSE_ERR_MISSING_VALUE:
+ if (current->argument->shortcut)
+ {
+ snprintf(buffer,
+ ARGPARSE_ERR_SIZE,
+ "argument -%c/--%s: expected one argument",
+ current->argument->shortcut,
+ current->argument->name);
+ }
+ else
+ {
+ snprintf(buffer,
+ ARGPARSE_ERR_SIZE,
+ "argument --%s: expected one argument",
+ current->argument->name);
+ }
+ break;
+ case ARGPARSE_ERR_UNRECOGNIZED_ARGUMENT:
+ snprintf(buffer,
+ ARGPARSE_ERR_SIZE,
+ "unrecognized argument: %s",
+ argv[argn]);
+ break;
+ case ARGPARSE_ERR_AMBIGUOUS_OPTION:
+ snprintf(buffer,
+ ARGPARSE_ERR_SIZE,
+ "ambiguous option: %s",
+ argv[argn]);
+ break;
+ case ARGPARSE_ERR_INVALID_CHOICE:
+ if (current->argument->shortcut)
+ {
+ snprintf(buffer,
+ ARGPARSE_ERR_SIZE,
+ "argument -%c/--%s: invalid choice: '%s'",
+ current->argument->shortcut,
+ current->argument->name,
+ argv[argn]);
+ }
+ else
+ {
+ snprintf(buffer,
+ ARGPARSE_ERR_SIZE,
+ "argument --%s: invalid choice: '%s'",
+ current->argument->name,
+ argv[argn]);
+ }
+ break;
+ case ARGPARSE_ERR_ARGUMENT_TOO_LONG:
+ snprintf(buffer,
+ ARGPARSE_ERR_SIZE,
+ "argument value exceeds character limit: %s",
+ current->argument->name);
+ break;
+ default:
+ *buffer = 0;
+ }
+ if (rc)
+ {
+ buffer[ARGPARSE_ERR_SIZE - 1] = '\0';
+ print_error(parser, buffer, bname);
+ quit(parser, EXIT_FAILURE);
+ }
+ }
+
+ /* when help is requested, print help and exit */
+ if (parser->show_help)
+ {
+ print_help(parser, bname);
+ quit(parser, EXIT_SUCCESS);
+ }
+
+ /* fill missing arguments with defaults */
+ fill_defaults(parser);
+
+ /* free parser */
+ argparse_free(parser);
+}
+
+static void quit(argparse_parser_t * parser, int rc)
+{
+ argparse_free(parser);
+ exit(rc);
+}
+
+static void fill_defaults(argparse_parser_t * parser)
+{
+ argparse_args_t * current;
+ for (current = parser->args; current != NULL; current = current->next)
+ {
+ switch (current->argument->action)
+ {
+ case ARGPARSE_STORE_TRUE:
+ case ARGPARSE_STORE_FALSE:
+ case ARGPARSE_STORE_INT:
+ if (current->argument->pt_value_int32_t == 0)
+ {
+ *current->argument->pt_value_int32_t =
+ current->argument->default_int32_t;
+ }
+ continue;
+ case ARGPARSE_STORE_STRING:
+ case ARGPARSE_STORE_STR_CHOICE:
+ if (!strlen(current->argument->str_value))
+ {
+ strcpy(
+ current->argument->str_value,
+ current->argument->str_default);
+ }
+ continue;
+ }
+ }
+}
+
+static void print_error(
+ argparse_parser_t * parser,
+ char * err_msg,
+ const char * bname)
+{
+ print_usage(parser, bname);
+ printf("%s: error: %s\n", bname, err_msg);
+}
+
+static void print_usage(argparse_parser_t * parser, const char * bname)
+{
+ char buffer[ARGPARSE_HELP_SIZE];
+ char uname[ARGPARSE_MAX_LEN_ARG];
+ size_t line_size;
+ size_t ident;
+ argparse_args_t * current;
+
+ snprintf(buffer, ARGPARSE_HELP_SIZE, "usage: %s ", bname);
+ line_size = ident = strlen(buffer);
+ printf("%s", buffer);
+ for (current = parser->args; current != NULL; current = current->next)
+ {
+ *buffer = 0;
+ switch (current->argument->action)
+ {
+ case ARGPARSE_STORE_TRUE:
+ case ARGPARSE_STORE_FALSE:
+ if (current->argument->shortcut)
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ "[-%c] ",
+ current->argument->shortcut);
+ }
+ else
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ "[--%s] ",
+ current->argument->name);
+ }
+ break;
+ case ARGPARSE_STORE_STRING:
+ case ARGPARSE_STORE_INT:
+ strcpy(uname, current->argument->name);
+ xstr_replace_char(uname, '-', '_');
+ xstr_upper_case(uname);
+ if (current->argument->shortcut)
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ "[-%c %s] ",
+ current->argument->shortcut,
+ uname);
+ }
+ else
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ "[--%s %s] ",
+ current->argument->name,
+ uname);
+ }
+ break;
+ case ARGPARSE_STORE_STR_CHOICE:
+ if (current->argument->shortcut)
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ "[-%c {%s}] ",
+ current->argument->shortcut,
+ current->argument->choices);
+ }
+ else
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ "[--%s {%s}] ",
+ current->argument->name,
+ current->argument->choices);
+ }
+ break;
+ }
+ buffer[ARGPARSE_HELP_SIZE - 1] = '\0';
+ line_size += strlen(buffer);
+ if (line_size > HELP_WIDTH)
+ {
+ printf("\n%*c%s", (int) ident, ' ', buffer);
+ line_size = ident + strlen(buffer);
+ }
+ else
+ {
+ printf("%s", buffer);
+ }
+ }
+ printf("\n");
+}
+
+static void print_help(argparse_parser_t * parser, const char * bname)
+{
+ char buffer[ARGPARSE_HELP_SIZE];
+ char uname[ARGPARSE_MAX_LEN_ARG];
+ argparse_args_t * current;
+ size_t line_size;
+
+ print_usage(parser, bname);
+ printf("\noptional arguments:\n");
+
+ current = parser->args;
+ for (;current != NULL; current = current->next)
+ {
+ *buffer = 0;
+ switch (current->argument->action)
+ {
+ case ARGPARSE_STORE_TRUE:
+ case ARGPARSE_STORE_FALSE:
+ if (current->argument->shortcut)
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ " -%c, --%s",
+ current->argument->shortcut,
+ current->argument->name);
+ }
+ else
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ " --%s",
+ current->argument->name);
+ }
+ break;
+ case ARGPARSE_STORE_STRING:
+ case ARGPARSE_STORE_INT:
+ strcpy(uname, current->argument->name);
+ xstr_replace_char(uname, '-', '_');
+ xstr_upper_case(uname);
+ if (current->argument->shortcut)
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ " -%c, --%s %s",
+ current->argument->shortcut,
+ current->argument->name,
+ uname);
+ }
+ else
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ " --%s %s",
+ current->argument->name,
+ uname);
+ }
+ break;
+ case ARGPARSE_STORE_STR_CHOICE:
+ if (current->argument->shortcut)
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ " -%c, --%s {%s},",
+ current->argument->shortcut,
+ current->argument->name,
+ current->argument->choices);
+ }
+ else
+ {
+ snprintf(buffer, ARGPARSE_HELP_SIZE,
+ " --%s {%s}",
+ current->argument->name,
+ current->argument->choices);
+ }
+ break;
+ }
+ buffer[ARGPARSE_HELP_SIZE - 1] = '\0';
+ line_size = strlen(buffer);
+ if (line_size > 24)
+ {
+ printf("%s\n%*c", buffer, 24, ' ');
+ }
+ else
+ {
+ printf("%-*s", 24, buffer);
+ }
+
+ printf("%s\n", current->argument->help);
+ }
+}
+
+static uint16_t get_argument(
+ argparse_args_t ** target,
+ argparse_parser_t * parser,
+ char * argument)
+{
+ char buffer[ARGPARSE_MAX_LEN_ARG];
+ uint16_t nfound = 0;
+ argparse_args_t * current;
+ for (current = parser->args; current != NULL; current = current->next)
+ {
+ if (current->argument->shortcut)
+ {
+ sprintf(buffer, "-%c", current->argument->shortcut);
+ if (strncmp(argument, buffer, strlen(argument)) == 0)
+ {
+ nfound++;
+ *target = current;
+ continue;
+ }
+ }
+ sprintf(buffer, "--%s", current->argument->name);
+ if (strncmp(argument, buffer, strlen(argument)) == 0)
+ {
+ nfound++;
+ *target = current;
+ }
+ }
+ return nfound;
+}
+
+static int process_arg(
+ argparse_parser_t * parser,
+ argparse_args_t ** arg,
+ int * argn,
+ int argc,
+ char * argv[])
+{
+ char buffer[ARGPARSE_MAX_LEN_ARG];
+ uint16_t num_matches;
+
+ /* get arg and get the number of matches. (should be exactly one match) */
+ num_matches = get_argument(&(*arg), parser, argv[*argn]);
+
+ /* check if we have exactly one match */
+ if (num_matches == 0)
+ {
+ return ARGPARSE_ERR_UNRECOGNIZED_ARGUMENT;
+ }
+ if (num_matches > 1)
+ {
+ return ARGPARSE_ERR_AMBIGUOUS_OPTION;
+ }
+
+ /* store true/false are not expecting more, they can be handled here */
+ if ( (*arg)->argument->action == ARGPARSE_STORE_TRUE ||
+ (*arg)->argument->action == ARGPARSE_STORE_FALSE)
+ {
+ *(*arg)->argument->pt_value_int32_t =
+ (*arg)->argument->action == ARGPARSE_STORE_TRUE;
+ return ARGPARSE_SUCCESS;
+ }
+
+ /* we expect a value an this value should not start with - */
+ if (++(*argn) == argc || *argv[*argn] == '-')
+ {
+ return ARGPARSE_ERR_MISSING_VALUE;
+ }
+
+ if (strlen(argv[*argn]) >= ARGPARSE_MAX_LEN_ARG)
+ {
+ return ARGPARSE_ERR_ARGUMENT_TOO_LONG;
+ }
+
+ /* create a copy from the value into a buffer */
+ strcpy(buffer, argv[*argn]);
+
+ /* handle action */
+ switch ((*arg)->argument->action)
+ {
+ case ARGPARSE_STORE_STRING:
+ strcpy((*arg)->argument->str_value, buffer);
+ return ARGPARSE_SUCCESS;
+ case ARGPARSE_STORE_INT:
+ *(*arg)->argument->pt_value_int32_t = (int32_t) atoi(buffer);
+ return ARGPARSE_SUCCESS;
+ case ARGPARSE_STORE_STR_CHOICE:
+ if (match_choice((*arg)->argument->choices, buffer))
+ {
+ strcpy((*arg)->argument->str_value, buffer);
+ return ARGPARSE_SUCCESS;
+ }
+ return ARGPARSE_ERR_INVALID_CHOICE;
+ default:
+ return ARGPARSE_ERR_UNHANDLED;
+ }
+
+}
+
+static bool match_choice(char * choices, char * choice)
+{
+ size_t len_rest = strlen(choices);
+ size_t len_choice = strlen(choice);
+ bool check = true;
+ char * walk = choices;
+
+ for (; walk; len_rest--, walk++)
+ {
+ /* quit if not enough left to match choice */
+ if (len_rest < len_choice)
+ {
+ return false;
+ }
+
+ /* when we should check, perform the check */
+ if ( check &&
+ strncmp(walk, choice, len_choice) == 0 &&
+ (len_rest == len_choice || walk[len_choice] == ','))
+ {
+ return true;
+ }
+
+ /* set check true again when a comma is found */
+ check = *walk == ',';
+ }
+ return false;
+}
--- /dev/null
+/*
+ * base64.c - Base64 encoding/decoding
+ */
+#include <stdlib.h>
+
+
+static const int base64__idx[256] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57,
+ 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6,
+ 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
+ 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
+ 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51
+};
+
+static const unsigned char base64__table[65] =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+char * base64_decode(const void * data, size_t n, size_t * size)
+{
+ const unsigned char * p = data;
+ int pad = n > 0 && (n % 4 || p[n - 1] == '=');
+ size_t i, j, L = ((n + 3) / 4 - pad) * 4;
+ char * out = malloc((L / 4 * 3) + (pad
+ ? (n > 1 && (n % 4 == 3 || p[n - 2] != '=')) + 2
+ : 1));
+ if (!out)
+ return NULL;
+
+ for (i = 0, j = 0; i < L; i += 4)
+ {
+ int nn = base64__idx[p[i]] << 18 | \
+ base64__idx[p[i + 1]] << 12 | \
+ base64__idx[p[i + 2]] << 6 | \
+ base64__idx[p[i + 3]];
+ out[j++] = nn >> 16;
+ out[j++] = nn >> 8 & 0xFF;
+ out[j++] = nn & 0xFF;
+ }
+
+ if (pad)
+ {
+ int nn = base64__idx[p[L]] << 18 | base64__idx[p[L + 1]] << 12;
+ out[j++] = nn >> 16;
+
+ if (n > L + 2 && p[L + 2] != '=')
+ {
+ nn |= base64__idx[p[L + 2]] << 6;
+ out[j++] = nn >> 8 & 0xFF;
+ }
+ }
+
+ *size = j;
+ out[*size] = '\0';
+ return out;
+}
+
+
+char * base64_encode(const void * data, size_t n, size_t * size)
+{
+ unsigned char * out;
+ unsigned char * pos;
+ const unsigned char * end, * in;
+ size_t olen = 4 * ((n + 2) / 3); /* 3-byte blocks to 4-byte */
+
+ if (olen < n || !(out = malloc(olen + 1)))
+ /* integer overflow or allocation error */
+ return NULL;
+
+ end = data + n;
+ in = data;
+ pos = out;
+
+ while (end - in >= 3)
+ {
+ *pos++ = base64__table[in[0] >> 2];
+ *pos++ = base64__table[((in[0] & 0x03) << 4) | (in[1] >> 4)];
+ *pos++ = base64__table[((in[1] & 0x0f) << 2) | (in[2] >> 6)];
+ *pos++ = base64__table[in[2] & 0x3f];
+ in += 3;
+ }
+
+ if (end - in)
+ {
+ *pos++ = base64__table[in[0] >> 2];
+ if (end - in == 1)
+ {
+ *pos++ = base64__table[(in[0] & 0x03) << 4];
+ *pos++ = '=';
+ }
+ else
+ {
+ *pos++ = base64__table[((in[0] & 0x03) << 4) | (in[1] >> 4)];
+ *pos++ = base64__table[(in[1] & 0x0f) << 2];
+ }
+ *pos++ = '=';
+ }
+
+ *size = pos - out;
+ out[*size] = '\0';
+ return (char *) out;
+}
+
+
--- /dev/null
+/*
+ * cexpr.c - Conditional expressions.
+ */
+#include <cexpr/cexpr.h>
+#include <assert.h>
+#include <stddef.h>
+#include <math.h>
+#include <siri/grammar/grammar.h>
+#include <siri/grammar/gramp.h>
+#include <logger/logger.h>
+#include <siri/db/series.h>
+#include <siri/db/shard.h>
+#include <siri/db/access.h>
+#include <xstr/xstr.h>
+
+#define VIA_NULL 0
+#define VIA_CEXPR 1
+#define VIA_COND 2
+
+#define EXPECTING_PROP 0
+#define EXPECTING_OPERATOR 1
+#define EXPECTING_VAL 2
+#define EXPECTING_NEXT 3
+
+/* make sure the condition is set to NULL because we will free
+ * the pointer in case CEXPR_MAX_CURLY_DEPT occurs.
+ */
+#define SET_CONDITION_AND_RETURN \
+ CEXPR_push_condition(cexpr, *condition); \
+ *condition = NULL; \
+ (*expecting) = EXPECTING_NEXT; \
+ return cexpr;
+
+static cexpr_t * CEXPR_new(void);
+static cexpr_condition_t * CEXPR_condition_new(void);
+static void CEXPR_condition_free(cexpr_condition_t * cond);
+static void CEXPR_push_condition(cexpr_t * cexpr, cexpr_condition_t * cond);
+static cexpr_t * CEXPR_push_and(cexpr_t * cexpr);
+static cexpr_t * CEXPR_push_or(cexpr_t * cexpr, cexpr_list_t * list);
+static cexpr_t * CEXPR_open_curly(cexpr_t * cexpr, cexpr_list_t * list);
+static cexpr_t * CEXPR_close_curly(cexpr_list_t * list);
+
+static cexpr_t * CEXPR_walk_node(
+ cleri_node_t * node,
+ cexpr_t * cexpr,
+ cexpr_list_t * list,
+ cexpr_condition_t ** condition,
+ int * expecting);
+
+cexpr_t * cexpr_from_node(cleri_node_t * node)
+{
+ cexpr_t * tmp;
+ cexpr_t * cexpr = CEXPR_new();
+ if (cexpr == NULL)
+ {
+ return NULL;
+ }
+
+ int expecting = EXPECTING_PROP;
+ cexpr_condition_t * condition = CEXPR_condition_new();
+
+ if (condition == NULL)
+ {
+ free(cexpr);
+ return NULL;
+ }
+
+ /* create a list, we only need this list while building an expression */
+ cexpr_list_t list;
+ list.len = 0;
+
+ /* by simply wrapping the expression between curly brackets we make sure
+ * we start validating at the correct start.
+ */
+ tmp = CEXPR_open_curly(cexpr, &list);
+
+ if (tmp == NULL)
+ {
+ free(cexpr);
+ free(condition);
+ return NULL;
+ }
+
+ /* build the expression */
+ cexpr = CEXPR_walk_node(node, tmp, &list, &condition, &expecting);
+
+ /* test if successful */
+ if (cexpr != NULL)
+ {
+ /* successful */
+ cexpr = CEXPR_close_curly(&list);
+ assert (list.len == 0 && condition == NULL);
+ }
+ else if (condition != NULL)
+ {
+ /* CEXPR_MAX_CURLY_DEPT, logging is done by the listener */
+ cexpr_free(list.cexpr[0]);
+ CEXPR_condition_free(condition);
+ }
+
+ return cexpr;
+}
+
+/*
+ * Returns 0 or 1 (false or true).
+ */
+int cexpr_int_cmp(
+ const cexpr_operator_t operator,
+ const int64_t a,
+ const int64_t b)
+{
+ switch (operator)
+ {
+ case CEXPR_EQ:
+ return a == b;
+ case CEXPR_NE:
+ return a != b;
+ case CEXPR_GT:
+ return a > b;
+ case CEXPR_LT:
+ return a < b;
+ case CEXPR_GE:
+ return a >= b;
+ case CEXPR_LE:
+ return a <= b;
+ default:
+ log_critical("Got an unexpected operator (int type): %d", operator);
+ assert (0);
+ }
+ /* we should NEVER get here */
+ return -1;
+}
+
+/*
+ * Returns 0 or 1 (false or true).
+ */
+int cexpr_double_cmp(
+ const cexpr_operator_t operator,
+ const double a,
+ const double b)
+{
+ switch (operator)
+ {
+ case CEXPR_EQ:
+ return isnan(a) ? isnan(b) : a == b;
+ case CEXPR_NE:
+ return isnan(a) ? !isnan(b) : a != b;
+ case CEXPR_GT:
+ return a > b;
+ case CEXPR_LT:
+ return a < b;
+ case CEXPR_GE:
+ return isnan(a) ? isnan(b) : a >= b;
+ case CEXPR_LE:
+ return isnan(a) ? isnan(b) : a <= b;
+ default:
+ log_critical("Got an unexpected operator (double type): %d", operator);
+ assert (0);
+ }
+ /* we should NEVER get here */
+ return -1;
+}
+
+/*
+ * Returns 0 or 1 (false or true).
+ */
+int cexpr_str_cmp(
+ const cexpr_operator_t operator,
+ const char * a,
+ const char * b)
+{
+ /* both a and b MUST be terminated strings */
+ assert (a != NULL && b != NULL);
+
+ switch (operator)
+ {
+ case CEXPR_EQ:
+ return strcmp(a, b) == 0;
+ case CEXPR_NE:
+ return strcmp(a, b) != 0;
+ case CEXPR_GT:
+ return strcmp(a, b) > 0;
+ case CEXPR_LT:
+ return strcmp(a, b) < 0;
+ case CEXPR_GE:
+ return strcmp(a, b) >= 0;
+ case CEXPR_LE:
+ return strcmp(a, b) <= 0;
+ case CEXPR_IN:
+ return strstr(a, b) != NULL;
+ case CEXPR_NI:
+ return strstr(a, b) == NULL;
+ default:
+ log_critical("Got an unexpected operator (string type): %d", operator);
+ assert (0);
+ }
+ /* we should NEVER get here */
+ return -1;
+}
+
+/*
+ * Returns 0 or 1 (false or true).
+ */
+int cexpr_bool_cmp(
+ const cexpr_operator_t operator,
+ const int64_t a,
+ const int64_t b)
+{
+ assert ((a == 0 || a == 1) && (b == 0 || b == 1));
+
+ switch (operator)
+ {
+ case CEXPR_EQ:
+ return a == b;
+ case CEXPR_NE:
+ return a != b;
+ default:
+ log_critical("Got an unexpected operator (boolean type): %d", operator);
+ assert (0);
+ }
+ /* we should NEVER get here */
+ return -1;
+}
+
+/*
+ * Returns 0 or 1 (false or true).
+ */
+int cexpr_run(cexpr_t * cexpr, cexpr_cb_t cb, void * obj)
+{
+ switch (cexpr->operator)
+ {
+ case CEXPR_AND:
+ /* tp_a cannot be VIA_NULL, but tp_b can */
+ assert (cexpr->tp_a != VIA_NULL);
+ return ((cexpr->tp_a == VIA_CEXPR) ?
+ cexpr_run(cexpr->via_a.cexpr, cb, obj) :
+ cb(obj, cexpr->via_a.cond)) &&
+ (cexpr->tp_b == VIA_NULL || ((cexpr->tp_b == VIA_CEXPR) ?
+ cexpr_run(cexpr->via_b.cexpr, cb, obj) :
+ cb(obj, cexpr->via_b.cond)));
+ case CEXPR_OR:
+ /* both tp_a and tp_b can NEVER be VIA_NULL */
+ assert (cexpr->tp_a != VIA_NULL && cexpr->tp_b != VIA_NULL);
+ return ((cexpr->tp_a == VIA_CEXPR) ?
+ cexpr_run(cexpr->via_a.cexpr, cb, obj) :
+ cb(obj, cexpr->via_a.cond)) ||
+ ((cexpr->tp_b == VIA_CEXPR) ?
+ cexpr_run(cexpr->via_b.cexpr, cb, obj) :
+ cb(obj, cexpr->via_b.cond));
+ default:
+ log_critical("operator must be AND or OR, got: %d", cexpr->operator);
+ }
+ assert (0);
+ return -1; /* this should NEVER happen */
+}
+
+int cexpr_contains(cexpr_t * cexpr, cexpr_cb_prop_t cb)
+{
+ /* should return either 1 or 0. (true or false) */
+
+ if (cexpr->tp_a == VIA_CEXPR && cexpr_contains(cexpr->via_a.cexpr, cb))
+ {
+ return 1;
+ }
+ if (cexpr->tp_b == VIA_CEXPR && cexpr_contains(cexpr->via_b.cexpr, cb))
+ {
+ return 1;
+ }
+ if (cexpr->tp_a == VIA_COND && cb(cexpr->via_a.cond->prop))
+ {
+ return 1;
+ }
+ if (cexpr->tp_b == VIA_COND && cb(cexpr->via_b.cond->prop))
+ {
+ return 1;
+ }
+ return 0;
+}
+
+void cexpr_free(cexpr_t * cexpr)
+{
+ switch (cexpr->tp_a)
+ {
+ case VIA_CEXPR: cexpr_free(cexpr->via_a.cexpr); break;
+ case VIA_COND: CEXPR_condition_free(cexpr->via_a.cond); break;
+ }
+
+ switch (cexpr->tp_b)
+ {
+ case VIA_CEXPR: cexpr_free(cexpr->via_b.cexpr); break;
+ case VIA_COND: CEXPR_condition_free(cexpr->via_b.cond); break;
+ }
+
+ free(cexpr);
+}
+
+/*
+ * Get operator from cleri node
+ */
+cexpr_operator_t cexpr_operator_fn(cleri_node_t * node)
+{
+ if (node->len == 1)
+ {
+ switch (*node->str)
+ {
+ case '>': return CEXPR_GT;
+ case '<': return CEXPR_LT;
+ case '~': return CEXPR_IN;
+ }
+ }
+ else
+ {
+ assert (node->len == 2);
+ switch (*node->str)
+ {
+ case '=': return CEXPR_EQ;
+ case '!': return (*(node->str + 1) == '=') ?
+ CEXPR_NE : CEXPR_NI;
+ case '>': return CEXPR_GE;
+ case '<': return CEXPR_LE;
+ }
+ }
+
+ assert (0);
+ return 0;
+}
+
+/*
+ * Return NULL in case of an error. (this can be when max_curly depth is
+ * reached or when a memory allocation error has occurred.
+ */
+static cexpr_t * CEXPR_walk_node(
+ cleri_node_t * node,
+ cexpr_t * cexpr,
+ cexpr_list_t * list,
+ cexpr_condition_t ** condition,
+ int * expecting)
+{
+ cexpr_condition_t * tmp_condition;
+ switch (*expecting)
+ {
+ case EXPECTING_PROP:
+ switch (node->cl_obj->tp)
+ {
+ case CLERI_TP_TOKEN:
+ /* this must be an open curly */
+ return CEXPR_open_curly(cexpr, list);
+
+ case CLERI_TP_KEYWORD:
+ /* this is a property we are looking for */
+ (*condition)->prop = node->cl_obj->gid;
+ (*expecting) = EXPECTING_OPERATOR;
+ return cexpr;
+
+ default:
+ /* this is probably a CHOICE */
+ break;
+ }
+ /* its fine to get here, not all cases are captured */
+ break;
+
+ case EXPECTING_OPERATOR:
+ switch (node->cl_obj->tp)
+ {
+ case CLERI_TP_TOKENS:
+ /* this one of the following operators:
+ * == < <= > >= != ~ !~
+ */
+ (*condition)->operator = cexpr_operator_fn(node);
+ (*expecting) = EXPECTING_VAL;
+ return cexpr;
+
+ default:
+ log_critical(
+ "Got an unexpected operator type: %d", node->cl_obj->tp);
+ }
+ assert (0);
+ break;
+
+ case EXPECTING_VAL:
+ switch (node->cl_obj->tp)
+ {
+ case CLERI_TP_RULE:
+ /* this is an integer or time expression, we can set the result
+ * and the condition.
+ */
+ (*condition)->int64 = CLERI_NODE_DATA(node);
+ SET_CONDITION_AND_RETURN
+ case CLERI_TP_CHOICE:
+ /* in case of a string, set the value and return */
+ if (node->cl_obj->gid == CLERI_GID_STRING)
+ {
+ (*condition)->str = malloc(node->len -1);
+ if ((*condition)->str == NULL)
+ {
+ return NULL;
+ }
+ xstr_extract_string((*condition)->str, node->str, node->len);
+ SET_CONDITION_AND_RETURN
+ }
+ /* can be a choice between keywords, in that case just wait */
+ break;
+ case CLERI_TP_KEYWORD:
+ /* for some keywords we can do some work to speed up checks */
+ switch (node->cl_obj->gid)
+ {
+ /* map boolean types */
+ case CLERI_GID_K_TRUE:
+ (*condition)->int64 = 1; break;
+ case CLERI_GID_K_FALSE:
+ (*condition)->int64 = 0; break;
+
+ /* map series types */
+ case CLERI_GID_K_INTEGER:
+ (*condition)->int64 = TP_INT; break;
+ case CLERI_GID_K_FLOAT:
+ (*condition)->int64 = TP_DOUBLE; break;
+ case CLERI_GID_K_STRING:
+ (*condition)->int64 = TP_STRING; break;
+
+ /* map shard types */
+ case CLERI_GID_K_NUMBER:
+ (*condition)->int64 = SIRIDB_SHARD_TP_NUMBER; break;
+ case CLERI_GID_K_LOG:
+ (*condition)->int64 = SIRIDB_SHARD_TP_LOG; break;
+
+ /* map access types */
+ case CLERI_GID_K_SHOW:
+ (*condition)->int64 = SIRIDB_ACCESS_SHOW; break;
+ case CLERI_GID_K_COUNT:
+ (*condition)->int64 = SIRIDB_ACCESS_COUNT; break;
+ case CLERI_GID_K_LIST:
+ (*condition)->int64 = SIRIDB_ACCESS_LIST; break;
+ case CLERI_GID_K_SELECT:
+ (*condition)->int64 = SIRIDB_ACCESS_SELECT; break;
+ case CLERI_GID_K_INSERT:
+ (*condition)->int64 = SIRIDB_ACCESS_INSERT; break;
+ case CLERI_GID_K_CREATE:
+ (*condition)->int64 = SIRIDB_ACCESS_CREATE; break;
+ case CLERI_GID_K_ALTER:
+ (*condition)->int64 = SIRIDB_ACCESS_ALTER; break;
+ case CLERI_GID_K_DROP:
+ (*condition)->int64 = SIRIDB_ACCESS_DROP; break;
+ case CLERI_GID_K_GRANT:
+ (*condition)->int64 = SIRIDB_ACCESS_GRANT; break;
+ case CLERI_GID_K_REVOKE:
+ (*condition)->int64 = SIRIDB_ACCESS_REVOKE; break;
+
+ /* map access profiles */
+ case CLERI_GID_K_READ:
+ (*condition)->int64 = SIRIDB_ACCESS_PROFILE_READ; break;
+ case CLERI_GID_K_WRITE:
+ (*condition)->int64 = SIRIDB_ACCESS_PROFILE_WRITE; break;
+ case CLERI_GID_K_MODIFY:
+ (*condition)->int64 = SIRIDB_ACCESS_PROFILE_MODIFY; break;
+ case CLERI_GID_K_FULL:
+ (*condition)->int64 = SIRIDB_ACCESS_PROFILE_FULL; break;
+
+ /* map log levels */
+ case CLERI_GID_K_DEBUG:
+ (*condition)->int64 = LOGGER_DEBUG; break;
+ case CLERI_GID_K_INFO:
+ (*condition)->int64 = LOGGER_INFO; break;
+ case CLERI_GID_K_WARNING:
+ (*condition)->int64 = LOGGER_WARNING; break;
+ case CLERI_GID_K_ERROR:
+ (*condition)->int64 = LOGGER_ERROR; break;
+ case CLERI_GID_K_CRITICAL:
+ (*condition)->int64 = LOGGER_CRITICAL; break;
+
+ default:
+ (*condition)->int64 = node->cl_obj->gid;
+ }
+ SET_CONDITION_AND_RETURN
+ default:
+ log_critical(
+ "Got an unexpected value type: %d", node->cl_obj->tp);
+ assert (0);
+ }
+ /* we allow to get here, not all cases return */
+ break;
+
+ case EXPECTING_NEXT:
+ /* this can be 'and', 'or', or an closing curly */
+ switch (node->cl_obj->tp)
+ {
+ case CLERI_TP_KEYWORD:
+ switch (node->cl_obj->gid)
+ {
+ case CLERI_GID_K_AND:
+ cexpr = CEXPR_push_and(cexpr);
+ if (cexpr == NULL)
+ {
+ return NULL;
+ }
+ break;
+ case CLERI_GID_K_OR:
+ cexpr = CEXPR_push_or(cexpr, list);
+ if (cexpr == NULL)
+ {
+ return NULL;
+ }
+ break;
+ default:
+ log_critical(
+ "Only 'and' or 'or' keywords are expected, got type: %"
+ PRIu32,
+ node->cl_obj->gid);
+ assert (0);
+ }
+ tmp_condition = CEXPR_condition_new();
+ if (tmp_condition == NULL)
+ {
+ return NULL;
+ }
+ *condition = tmp_condition;
+ *expecting = EXPECTING_PROP;
+ return cexpr;
+
+ case CLERI_TP_TOKEN:
+ return CEXPR_close_curly(list);
+
+ default:
+ log_critical(
+ "Only and/or/closing curly are expected. got type: %d",
+ node->cl_obj->tp);
+ }
+ /* we must NEVER get here */
+ assert (0);
+ break;
+ }
+
+ cleri_children_t * current = node->children;
+
+ while (current != NULL && current->node != NULL)
+ {
+ cexpr = CEXPR_walk_node(
+ current->node,
+ cexpr,
+ list,
+ condition,
+ expecting);
+ if (cexpr == NULL)
+ {
+ return NULL;
+ }
+ current = current->next;
+ }
+
+ return cexpr;
+}
+
+/*
+ * Returns NULL in case of an error
+ */
+static cexpr_t * CEXPR_new(void)
+{
+ cexpr_t * cexpr = malloc(sizeof(cexpr_t));
+ if (cexpr != NULL)
+ {
+ cexpr->operator = CEXPR_AND;
+ cexpr->tp_a = VIA_NULL;
+ cexpr->tp_b = VIA_NULL;
+ cexpr->via_a.cexpr = NULL;
+ }
+ return cexpr;
+}
+
+/*
+ * Returns NULL in case of an error
+ */
+static cexpr_condition_t * CEXPR_condition_new(void)
+{
+ cexpr_condition_t * condition = malloc(sizeof(cexpr_condition_t));
+
+ if (condition != NULL)
+ {
+ condition->int64 = 0;
+ condition->str = NULL;
+ }
+
+ return condition;
+}
+
+static void CEXPR_condition_free(cexpr_condition_t * cond)
+{
+ free(cond->str);
+ free(cond);
+}
+
+/*
+ * Returns NULL in case or an error
+ */
+static cexpr_t * CEXPR_push_and(cexpr_t * cexpr)
+{
+ if (cexpr->tp_b == VIA_NULL)
+ {
+ return cexpr;
+ }
+ cexpr_t * new_cexpr = CEXPR_new();
+
+ if (new_cexpr != NULL)
+ {
+ new_cexpr->tp_a = cexpr->tp_b;
+ new_cexpr->via_a = cexpr->via_b;
+ cexpr->tp_b = VIA_CEXPR;
+ cexpr->via_b.cexpr = new_cexpr;
+ }
+ return new_cexpr;
+}
+
+/*
+ * Returns NULL in case or an error
+ */
+static cexpr_t * CEXPR_push_or(cexpr_t * cexpr, cexpr_list_t * list)
+{
+ if (cexpr->tp_b == VIA_NULL)
+ {
+ /* this can happen only at the first condition */
+ cexpr->operator = CEXPR_OR;
+ return cexpr;
+ }
+
+ cexpr_t * new_cexpr = CEXPR_new();
+
+ if (new_cexpr != NULL)
+ {
+ size_t selected = list->len - 1;
+ cexpr = list->cexpr[selected];
+
+
+ new_cexpr->operator = CEXPR_OR;
+ new_cexpr->tp_a = VIA_CEXPR;
+ new_cexpr->via_a.cexpr = cexpr;
+
+ list->cexpr[selected] = new_cexpr;
+ }
+
+ return new_cexpr;
+}
+
+static void CEXPR_push_condition(cexpr_t * cexpr, cexpr_condition_t * cond)
+{
+ if (cexpr->tp_a == VIA_NULL)
+ {
+ cexpr->tp_a = VIA_COND;
+ cexpr->via_a.cond = cond;
+ }
+ else
+ {
+ assert(cexpr->tp_b == VIA_NULL);
+ cexpr->tp_b = VIA_COND;
+ cexpr->via_b.cond = cond;
+ }
+}
+
+static cexpr_t * CEXPR_open_curly(cexpr_t * cexpr, cexpr_list_t * list)
+{
+ if (list->len == CEXPR_MAX_CURLY_DEPTH)
+ {
+ /* max expression depth reached */
+ return NULL;
+ }
+
+ /* create new expression */
+ cexpr_t * new_cexpr = CEXPR_new();
+
+ if (new_cexpr == NULL)
+ {
+ /* memory allocation error */
+ return NULL;
+ }
+
+
+ /* save current cexpr in depth. */
+ list->cexpr[list->len] = cexpr;
+
+ if (cexpr->tp_a == VIA_NULL)
+ {
+ cexpr->tp_a = VIA_CEXPR;
+ cexpr->via_a.cexpr = new_cexpr;
+ }
+ else
+ {
+ assert(cexpr->tp_b == VIA_NULL);
+ cexpr->tp_b = VIA_CEXPR;
+ cexpr->via_b.cexpr = new_cexpr;
+ }
+ list->len++;
+ return new_cexpr;
+}
+
+static cexpr_t * CEXPR_close_curly(cexpr_list_t * list)
+{
+ assert (list->len > 0);
+ list->len--;
+ return list->cexpr[list->len];
+}
--- /dev/null
+/*
+ * cfgparser.c - Reading (and later writing) to INI style files.
+ */
+#include <cfgparser/cfgparser.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include <xstr/xstr.h>
+#include <siri/err.h>
+
+static void cfgparser_free_sections(cfgparser_section_t * root);
+static void cfgparser_free_options(cfgparser_option_t * root);
+static cfgparser_option_t * cfgparser_new_option(
+ cfgparser_section_t * section,
+ const char * name,
+ cfgparser_tp_t tp,
+ cfgparser_via_t * val,
+ cfgparser_via_t * def);
+
+#define MAXLINE 255
+
+/*
+ * Returns CFGPARSER_SUCCESS if successful or something else in case of an
+ * error.
+ *
+ * Note: In case of a memory allocation error a SIGNAL is raised but
+ * the returned error message might be incorrect. We do however
+ * never return CFGPARSER_SUCCESS when this happens.
+ */
+cfgparser_return_t cfgparser_read(cfgparser_t * cfgparser, const char * fn)
+{
+ FILE * fp;
+ char line[MAXLINE];
+ char * pt;
+ const char * name;
+ cfgparser_section_t * section = NULL;
+ cfgparser_option_t * option = NULL;
+ int found = 0;
+ double d;
+
+ fp = fopen(fn, "r");
+ if (fp == NULL)
+ {
+ return CFGPARSER_ERR_READING_FILE;
+ }
+
+
+ while (fgets(line, MAXLINE, fp) != NULL)
+ {
+ /* set pointer to line */
+ pt = line;
+
+ /* trims all whitespace */
+ xstr_trim(&pt, 0);
+
+ if (*pt == '#' || *pt == 0)
+ {
+ continue;
+ }
+
+ if (*pt == '[' && pt[strlen(pt) - 1] == ']')
+ {
+ xstr_trim(&pt, '[');
+ xstr_trim(&pt, ']');
+ section = cfgparser_section(cfgparser, pt);
+ if (section == NULL)
+ {
+ /* signal is raised */
+ fclose(fp);
+ return CFGPARSER_ERR_SESSION_NOT_OPEN;
+ }
+ continue;
+ }
+
+ if (section == NULL)
+ {
+ fclose(fp);
+ return CFGPARSER_ERR_SESSION_NOT_OPEN;
+ }
+
+ for (found = 0, name = pt; *pt; pt++)
+ {
+ if (isspace(*pt) || *pt == '=')
+ {
+ if (*pt == '=')
+ {
+ found = 1;
+ }
+ *pt = 0;
+ continue;
+ }
+ if (found)
+ {
+ break;
+ }
+ }
+
+ if (!found)
+ {
+ fclose(fp);
+ return CFGPARSER_ERR_MISSING_EQUAL_SIGN;
+ }
+
+
+ if (xstr_is_int(pt))
+ {
+ option = cfgparser_integer_option(section, name, atoi(pt), 0);
+ }
+ else if (xstr_is_float(pt))
+ {
+ sscanf(pt, "%lf", &d);
+ option = cfgparser_real_option(section, name, d, 0.0f);
+ }
+ else
+ {
+ option = cfgparser_string_option(section, name, pt, "");
+ }
+
+ if (option == NULL)
+ {
+ fclose(fp);
+ /*
+ * this could also be due to a allocation error, a SIGNAL is set
+ * in that case.
+ */
+ return CFGPARSER_ERR_OPTION_ALREADY_DEFINED;
+ }
+
+ }
+
+ fclose(fp);
+
+ return CFGPARSER_SUCCESS;
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+cfgparser_t * cfgparser_new(void)
+{
+ cfgparser_t * cfgparser;
+
+ cfgparser = malloc(sizeof(cfgparser_t));
+ if (cfgparser != NULL)
+ {
+ cfgparser->sections = NULL;
+ }
+ return cfgparser;
+}
+
+/*
+ * Destroy cfgparser. (parsing NULL is not allowed)
+ */
+void cfgparser_free(cfgparser_t * cfgparser)
+{
+ cfgparser_free_sections(cfgparser->sections);
+ free(cfgparser);
+}
+
+/*
+ * Returns a section from cfgparser. If the section does not exist, a new
+ * section will be created.
+ *
+ * In case of an error, NULL is returned and a SIGNAL is raised.
+ */
+cfgparser_section_t * cfgparser_section(
+ cfgparser_t * cfgparser,
+ const char * name)
+{
+ cfgparser_section_t * current = cfgparser->sections;
+
+ if (current == NULL)
+ {
+ cfgparser->sections = malloc(sizeof(cfgparser_section_t));
+ if (cfgparser->sections == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ cfgparser->sections->options = NULL;
+ cfgparser->sections->next = NULL;
+ cfgparser->sections->name = strdup(name);
+ if (cfgparser->sections->name == NULL)
+ {
+ ERR_ALLOC
+ free(cfgparser->sections);
+ cfgparser->sections = NULL;
+ }
+ }
+ return cfgparser->sections;
+ }
+
+ while (current->next != NULL)
+ {
+ if (strcmp(current->name, name) == 0)
+ {
+ return current;
+ }
+ current = current->next;
+ }
+ current->next = malloc(sizeof(cfgparser_section_t));
+ if (current->next == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ current->next->options = NULL;
+ current->next->next = NULL;
+ current->next->name = strdup(name);
+ if (current->next->name == NULL)
+ {
+ ERR_ALLOC
+ free(current->next);
+ current->next = NULL;
+ }
+ }
+ return current->next;
+}
+
+/*
+ * Creates and returns a new options. NULL is returned in case the option
+ * already existed.
+ *
+ * In case of an allocation error a SIGNAL is raises and NULL is returned.
+ */
+cfgparser_option_t * cfgparser_string_option(
+ cfgparser_section_t * section,
+ const char * name,
+ const char * val,
+ const char * def)
+{
+ cfgparser_via_t * val_u = malloc(sizeof(cfgparser_via_t));
+ cfgparser_via_t * def_u = malloc(sizeof(cfgparser_via_t));
+
+ if (val_u == NULL || def_u == NULL)
+ {
+ ERR_ALLOC
+ free(val_u);
+ free(def_u);
+ return NULL;
+ }
+
+ val_u->string = strdup(val);
+ def_u->string = strdup(def);
+
+ if (val_u->string == NULL || def_u->string == NULL)
+ {
+ ERR_ALLOC
+ free(val_u->string);
+ free(def_u->string);
+ free(val_u);
+ free(def_u);
+ return NULL;
+ }
+
+ return cfgparser_new_option(
+ section,
+ name,
+ CFGPARSER_TP_STRING,
+ val_u,
+ def_u);
+}
+
+/*
+ * Creates and returns a new options. NULL is returned in case the option
+ * already existed.
+ *
+ * In case of an allocation error a SIGNAL is raises and NULL is returned.
+ */
+cfgparser_option_t * cfgparser_integer_option(
+ cfgparser_section_t * section,
+ const char * name,
+ int32_t val,
+ int32_t def)
+{
+ cfgparser_via_t * val_u = malloc(sizeof(cfgparser_via_t));
+ cfgparser_via_t * def_u = malloc(sizeof(cfgparser_via_t));
+ if (val_u == NULL || def_u == NULL)
+ {
+ ERR_ALLOC
+ free(val_u);
+ free(def_u);
+ return NULL;
+ }
+ val_u->integer = val;
+ def_u->integer = def;
+ return cfgparser_new_option(
+ section,
+ name,
+ CFGPARSER_TP_INTEGER,
+ val_u,
+ def_u);
+}
+
+/*
+ * Creates and returns a new options. NULL is returned in case the option
+ * already existed.
+ *
+ * In case of an allocation error a SIGNAL is raises and NULL is returned.
+ */
+cfgparser_option_t * cfgparser_real_option(
+ cfgparser_section_t * section,
+ const char * name,
+ double val,
+ double def)
+{
+ cfgparser_via_t * val_u = malloc(sizeof(cfgparser_via_t));
+ cfgparser_via_t * def_u = malloc(sizeof(cfgparser_via_t));
+ if (val_u == NULL || def_u == NULL)
+ {
+ ERR_ALLOC
+ free(val_u);
+ free(def_u);
+ return NULL;
+ }
+ val_u->real = val;
+ def_u->real = def;
+ return cfgparser_new_option(
+ section,
+ name,
+ CFGPARSER_TP_REAL,
+ val_u,
+ def_u);
+}
+
+/*
+ * Returns an error message for a cfgparser_return_t code.
+ */
+const char * cfgparser_errmsg(cfgparser_return_t err)
+{
+ switch (err)
+ {
+ case CFGPARSER_SUCCESS:
+ return "configuration file parsed successfully";
+ case CFGPARSER_ERR_READING_FILE:
+ return "cannot open file for reading";
+ case CFGPARSER_ERR_SESSION_NOT_OPEN:
+ return "got a line without a section";
+ case CFGPARSER_ERR_MISSING_EQUAL_SIGN:
+ return "missing equal sign in at least one line";
+ case CFGPARSER_ERR_OPTION_ALREADY_DEFINED:
+ return "option defined twice within one section";
+ case CFGPARSER_ERR_SECTION_NOT_FOUND:
+ return "section not found";
+ case CFGPARSER_ERR_OPTION_NOT_FOUND:
+ return "option not found";
+ }
+ return "";
+}
+
+/*
+ * Finds a section in a cfgparser. When the result is CFGPARSER_SUCCESS,
+ * 'section' is set. In case CFGPARSER_ERR_SECTION_NOT_FOUND is returned,
+ * 'section' is untouched.
+ */
+cfgparser_return_t cfgparser_get_section(
+ cfgparser_section_t ** section,
+ cfgparser_t * cfgparser,
+ const char * section_name)
+{
+ cfgparser_section_t * current = cfgparser->sections;
+ while (current != NULL)
+ {
+ if (strcmp(current->name, section_name) == 0)
+ {
+ *section = current;
+ return CFGPARSER_SUCCESS;
+ }
+ current = current->next;
+ }
+ return CFGPARSER_ERR_SECTION_NOT_FOUND;
+}
+
+/*
+ * Finds a option in a cfgparser. When the result is CFGPARSER_SUCCESS,
+ * 'option' is set. In case CFGPARSER_ERR_SECTION_NOT_FOUND or
+ * CFGPARSER_ERR_OPTION_NOT_FOUND is returned, 'option' is untouched.
+ */
+cfgparser_return_t cfgparser_get_option(
+ cfgparser_option_t ** option,
+ cfgparser_t * cfgparser,
+ const char * section_name,
+ const char * option_name)
+{
+ cfgparser_option_t * current;
+ cfgparser_section_t * section;
+ cfgparser_return_t rc;
+
+ rc = cfgparser_get_section(§ion, cfgparser, section_name);
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ return rc;
+ }
+ current = section->options;
+ while (current != NULL)
+ {
+ if (strcmp(current->name, option_name) == 0)
+ {
+ *option = current;
+ return CFGPARSER_SUCCESS;
+ }
+ current = current->next;
+ }
+ return CFGPARSER_ERR_OPTION_NOT_FOUND;
+}
+
+/*
+ * Creates and returns a new option or NULL in case the option exists.
+ * Note that in this case 'val' and 'def' are destroyed.
+ *
+ * In case an error occurs, NULL is returned too and a SIGNAL is raised.
+ */
+static cfgparser_option_t * cfgparser_new_option(
+ cfgparser_section_t * section,
+ const char * name,
+ cfgparser_tp_t tp,
+ cfgparser_via_t * val,
+ cfgparser_via_t * def)
+{
+ cfgparser_option_t * current = section->options;
+ cfgparser_option_t * prev;
+
+ if (current == NULL)
+ {
+ section->options = malloc(sizeof(cfgparser_option_t));
+ if (section->options == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ section->options->tp = tp;
+ section->options->val = val;
+ section->options->def = def;
+ section->options->next = NULL;
+ section->options->name = strdup(name);
+ if (section->options->name == NULL)
+ {
+ ERR_ALLOC
+ free(section->options);
+ section->options = NULL;
+ }
+ }
+ return section->options;
+ }
+
+ while (current != NULL)
+ {
+ prev = current;
+ if (strcmp(current->name, name) == 0)
+ {
+ if (tp == CFGPARSER_TP_STRING)
+ {
+ free(val->string);
+ free(def->string);
+ }
+ free(val);
+ free(def);
+ return NULL;
+ }
+ current = current->next;
+ }
+
+ prev->next = malloc(sizeof(cfgparser_option_t));
+ if (prev->next == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ prev->next->tp = tp;
+ prev->next->val = val;
+ prev->next->def = def;
+ prev->next->next = NULL;
+ if ((prev->next->name = strdup(name)) == NULL)
+ {
+ ERR_ALLOC
+ free(prev->next);
+ prev->next = NULL;
+ }
+ }
+ return prev->next;
+}
+
+/*
+ * Destroy cfgparser_section_t. (parsing NULL is allowed)
+ */
+static void cfgparser_free_sections(cfgparser_section_t * root)
+{
+ cfgparser_section_t * next;
+
+ while (root != NULL)
+ {
+ next = root->next;
+ free(root->name);
+ cfgparser_free_options(root->options);
+ free(root);
+ root = next;
+ }
+}
+
+/*
+ * Destroy cfgparser_option_t. (parsing NULL is allowed)
+ */
+static void cfgparser_free_options(cfgparser_option_t * root)
+{
+ cfgparser_option_t * next;
+
+ while (root != NULL)
+ {
+ next = root->next;
+ if (root->tp == CFGPARSER_TP_STRING)
+ {
+ free(root->val->string);
+ free(root->def->string);
+ }
+ free(root->val);
+ free(root->def);
+ free(root->name);
+ free(root);
+ root = next;
+ }
+}
--- /dev/null
+/*
+ * ctree.c - Compact Binary Tree implementation.
+ */
+#include <ctree/ctree.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <assert.h>
+#include <logger/logger.h>
+
+/* initial buffer size, this is not fixed but can grow if needed */
+#define CT_BUF_SIZE 128
+#define BLOCKSZ 32
+
+static ct_node_t * CT_node_new(const char * key, size_t len, void * data);
+static int CT_node_resize(ct_node_t * node, uint8_t pos);
+static int CT_add(
+ ct_node_t * node,
+ const char * key,
+ void * data);
+static void * CT_pop(ct_node_t * parent, ct_node_t ** nd, const char * key);
+static void CT_dec_node(ct_node_t * node);
+static void CT_merge_node(ct_node_t * node);
+static int CT_items(
+ ct_node_t * node,
+ size_t len,
+ size_t buffer_sz,
+ char ** buffer,
+ ct_item_cb cb,
+ void * args);
+static int CT_values(
+ ct_node_t * node,
+ ct_val_cb cb,
+ void * args);
+static void CT_valuesn(
+ ct_node_t * node,
+ size_t * n,
+ ct_val_cb cb,
+ void * args);
+static void CT_free(ct_node_t * node, ct_free_cb cb);
+
+/*
+ * Returns NULL in case an error has occurred.
+ */
+ct_t * ct_new(void)
+{
+ ct_t * ct = malloc(sizeof(ct_t));
+ if (ct == NULL)
+ {
+ return NULL;
+ }
+
+ ct->len = 0;
+ ct->nodes = NULL;
+ ct->offset = UINT8_MAX;
+ ct->n = 0;
+
+ return ct;
+}
+
+/*
+ * Destroy ct-tree. Parsing NULL is NOT allowed.
+ * Call-back function will be called on each item in the tree.
+ */
+void ct_free(ct_t * ct, ct_free_cb cb)
+{
+ if (ct->nodes != NULL)
+ {
+ uint_fast16_t i, end;
+ for (i = 0, end = ct->n * BLOCKSZ; i < end; i++)
+ {
+ if ((*ct->nodes)[i] != NULL)
+ {
+ CT_free((*ct->nodes)[i], cb);
+ }
+ }
+ free(ct->nodes);
+ }
+ free(ct);
+}
+
+/*
+ * Add a new key/value. return CT_EXISTS (1) if the key already
+ * exists and CT_OK (0) if not. When the key exists the value will not
+ * be overwritten. CT_EXISTS is also returned when the key has length 0.
+ *
+ * In case of an error, CT_ERR (-1) will be returned.
+ */
+int ct_add(ct_t * ct, const char * key, void * data)
+{
+ int rc;
+ ct_node_t ** nd;
+ uint8_t k = (uint8_t) *key;
+ if (!*key)
+ {
+ return CT_EXISTS;
+ }
+
+ if (CT_node_resize((ct_node_t *) ct, k / BLOCKSZ))
+ {
+ return CT_ERR;
+ }
+
+ nd = &(*ct->nodes)[k - ct->offset * BLOCKSZ];
+ key++;
+
+ if (*nd != NULL)
+ {
+ rc = CT_add(*nd, key, data);
+ if (rc == CT_OK)
+ {
+ ct->len++;
+ }
+ }
+ else
+ {
+ *nd = CT_node_new(key, strlen(key), data);
+ if (*nd == NULL)
+ {
+ rc = CT_ERR;
+ }
+ else
+ {
+ ct->len++;
+ rc = CT_OK;
+ }
+ }
+ return rc;
+}
+
+/*
+ * Returns an item or NULL if the key does not exist.
+ */
+void * ct_get(ct_t * ct, const char * key)
+{
+ void ** data = ct_getaddr(ct, key);
+ return (data) ? *data : NULL;
+}
+
+/*
+ * Returns the address of an item or NULL if the key does not exist.
+ */
+void ** ct_getaddr(ct_t * ct, const char * key)
+{
+ ct_node_t * nd;
+ uint8_t k = (uint8_t) *key;
+ uint8_t pos = k / BLOCKSZ;
+
+ if (!*key || pos < ct->offset || pos >= ct->offset + ct->n)
+ {
+ return NULL;
+ }
+
+ nd = (*ct->nodes)[k - ct->offset * BLOCKSZ];
+
+ while (nd && !strncmp(nd->key, ++key, nd->len))
+ {
+ key += nd->len;
+
+ if (!*key) return &nd->data;
+ if (!nd->nodes) return NULL;
+
+ k = (uint8_t) *key;
+ pos = k / BLOCKSZ;
+
+ if (pos < nd->offset || pos >= nd->offset + nd->n)
+ {
+ return NULL;
+ }
+
+ nd = (*nd->nodes)[k - nd->offset * BLOCKSZ];
+ }
+
+ return NULL;
+}
+
+/*
+ * Returns an item or NULL if the key does not exist.
+ */
+void * ct_getn(ct_t * ct, const char * key, size_t n)
+{
+ size_t diff = 1;
+ ct_node_t * nd;
+ uint8_t k, pos;
+
+ if (!n)
+ {
+ return NULL;
+ }
+
+ k = (uint8_t) *key;
+ pos = k / BLOCKSZ;
+
+ if (pos < ct->offset || pos >= ct->offset + ct->n)
+ {
+ return NULL;
+ }
+
+ nd = (*ct->nodes)[k - ct->offset * BLOCKSZ];
+
+ while (nd)
+ {
+ key += diff;
+ n -= diff;
+
+ if (n < nd->len || strncmp(nd->key, key, nd->len))
+ {
+ return NULL;
+ }
+
+ if (nd->len == n) return nd->data;
+ if (!nd->nodes) return NULL;
+
+ k = (uint8_t) key[nd->len];
+ pos = k / BLOCKSZ;
+
+ if (pos < nd->offset || pos >= nd->offset + nd->n)
+ {
+ return NULL;
+ }
+
+ diff = nd->len + 1; /* n - diff is at least 0 */
+ nd = (*nd->nodes)[k - nd->offset * BLOCKSZ];
+ }
+
+ return NULL;
+}
+
+/*
+ * Removes and returns an item from the tree or NULL when not found.
+ *
+ * (re-allocation might fail but this is not critical)
+ */
+void * ct_pop(ct_t * ct, const char * key)
+{
+ ct_node_t ** nd;
+ void * data;
+ uint8_t k = (uint8_t) *key;
+ uint8_t pos = k / BLOCKSZ;
+
+ if (!*key || pos < ct->offset || pos >= ct->offset + ct->n)
+ {
+ data = NULL;
+ }
+ else
+ {
+ nd = &(*ct->nodes)[k - ct->offset * BLOCKSZ];
+
+ if (*nd == NULL)
+ {
+ data = NULL;
+ }
+ else
+ {
+ data = CT_pop(NULL, nd, key + 1);
+ if (data != NULL)
+ {
+ ct->len--;
+ }
+ }
+ }
+
+ return data;
+}
+
+/*
+ * Loop over all items in the tree and perform the call-back on each item.
+ *
+ * Looping stops on the first call-back returning a non-zero value.
+ *
+ * Returns 0 when the call-back is called on all items, -1 in case of
+ * an allocation error or 1 when looping did not finish because of a non
+ * zero return value.
+ */
+int ct_items(ct_t * ct, ct_item_cb cb, void * args)
+{
+ size_t buffer_sz = CT_BUF_SIZE;
+ size_t len = 1;
+ ct_node_t * nd;
+ int rc = 0;
+ char * buffer = malloc(buffer_sz);
+ uint_fast16_t i, end;
+
+ if (buffer == NULL)
+ {
+ return -1;
+ }
+ for (i = 0, end = ct->n * BLOCKSZ; !rc && i < end; i++)
+ {
+ if ((nd = (*ct->nodes)[i]) == NULL)
+ {
+ continue;
+ }
+ *buffer = (char) (i + ct->offset * BLOCKSZ);
+ rc = CT_items(nd, len, buffer_sz, &buffer, cb, args);
+ }
+ free(buffer);
+ return rc;
+}
+
+/*
+ * Loop over all values in the tree and perform the call-back on each value.
+ *
+ * Returns the sum of all the call-backs.
+ */
+int ct_values(ct_t * ct, ct_val_cb cb, void * args)
+{
+ ct_node_t * nd;
+ int rc = 0;
+ uint_fast16_t i, end;
+
+ for (i = 0, end = ct->n * BLOCKSZ; i < end; i++)
+ {
+ if ((nd = (*ct->nodes)[i]) == NULL)
+ {
+ continue;
+ }
+ rc += CT_values(nd, cb, args);
+ }
+
+ return rc;
+}
+
+/*
+ * Walking stops either when the call-back is called on each value or
+ * when 'n' is zero. 'n' will be decremented by the result of each call-back.
+ */
+void ct_valuesn(ct_t * ct, size_t * n, ct_val_cb cb, void * args)
+{
+ ct_node_t * nd;
+ uint_fast16_t i, end;
+
+ for (i = 0, end = ct->n * BLOCKSZ; *n && i < end; i++)
+ {
+ if ((nd = (*ct->nodes)[i]) == NULL)
+ {
+ continue;
+ }
+ CT_valuesn(nd, n, cb, args);
+ }
+}
+
+/*
+ * Loop over all items in the tree and perform the call-back on each item.
+ * Walking stops either when the call-back is called on each item or
+ * when a callback return a non zero value.
+ *
+ * Returns 0 when successful or -1 in case of an error or 1 if looping has
+ * stopped because a failed callback.
+ */
+static int CT_items(
+ ct_node_t * node,
+ size_t len,
+ size_t buffer_sz,
+ char ** buffer,
+ ct_item_cb cb,
+ void * args)
+{
+ if (node->len + len + 1 > buffer_sz)
+ {
+ char * tmp;
+ buffer_sz = ((node->len + len) / CT_BUF_SIZE + 1) * CT_BUF_SIZE;
+ tmp = (char *) realloc(*buffer, buffer_sz);
+ if (tmp == NULL)
+ {
+ return -1;
+ }
+ *buffer = tmp;
+ }
+
+ memcpy(*buffer + len, node->key, node->len);
+ len += node->len;
+
+ if (node->data != NULL && (*cb)(*buffer, len, node->data, args))
+ {
+ return 1;
+ }
+
+ if (node->nodes != NULL)
+ {
+ ct_node_t * nd;
+ int rc;
+ uint_fast16_t i, end;
+
+ for (i = 0, end = node->n * BLOCKSZ; i < end; i++)
+ {
+ if ((nd = (*node->nodes)[i]) == NULL)
+ {
+ continue;
+ }
+ *(*buffer + len) = (char) (i + node->offset * BLOCKSZ);
+ rc = CT_items(nd, len + 1, buffer_sz, buffer, cb, args);
+ if (rc)
+ {
+ return rc;
+ }
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Loop over all values in the tree and perform the call-back on each value.
+ *
+ * The value returned is the sum of all the call-backs.
+ */
+static int CT_values(
+ ct_node_t * node,
+ ct_val_cb cb,
+ void * args)
+{
+ int rc = 0;
+
+ if (node->data != NULL)
+ {
+ rc += (*cb)(node->data, args);
+ }
+
+ if (node->nodes != NULL)
+ {
+ ct_node_t * nd;
+ uint_fast16_t i, end;
+
+ for (i = 0, end = node->n * BLOCKSZ; i < end; i++)
+ {
+ if ((nd = (*node->nodes)[i]) == NULL)
+ {
+ continue;
+ }
+ rc += CT_values(nd, cb, args);
+ }
+ }
+
+ return rc;
+}
+
+/*
+ * Loop over all values in the tree and perform the call-back on each value.
+ * */
+static void CT_valuesn(
+ ct_node_t * node,
+ size_t * n,
+ ct_val_cb cb,
+ void * args)
+{
+ if (node->data != NULL)
+ {
+ *n -= (*cb)(node->data, args);
+ }
+
+ if (node->nodes != NULL)
+ {
+ ct_node_t * nd;
+ uint_fast16_t i, end;
+
+ for (i = 0, end = node->n * BLOCKSZ; *n && i < end; i++)
+ {
+ if ((nd = (*node->nodes)[i]) == NULL)
+ {
+ continue;
+ }
+ CT_valuesn(nd, n, cb, args);
+ }
+ }
+}
+
+/*
+ * Returns CT_OK when the item is added, CT_EXISTS if the item already exists,
+ * or CT_ERR in case or an error.
+ * In case of CT_EXISTS the existing item is not overwritten.
+ */
+static int CT_add(
+ ct_node_t * node,
+ const char * key,
+ void * data)
+{
+ size_t n;
+ for (n = 0; n < node->len; n++, key++)
+ {
+ char * pt = node->key + n;
+ if (*key != *pt)
+ {
+ size_t new_sz;
+ uint8_t k = (uint8_t) *pt;
+ ct_node_t * nd;
+
+ /* create new nodes */
+ ct_nodes_t * new_nodes =
+ (ct_nodes_t *) calloc(1, sizeof(ct_nodes_t));
+ if (new_nodes == NULL)
+ {
+ return CT_ERR;
+ }
+
+ /* create new nodes with rest of node pt */
+ nd = (*new_nodes)[k % BLOCKSZ] =
+ CT_node_new(pt + 1, node->len - n - 1, node->data);
+ if (nd == NULL)
+ {
+ return CT_ERR;
+ }
+
+ /* bind the -rest- of current node to the new nodes */
+ nd->nodes = node->nodes;
+ nd->size = node->size;
+ nd->offset = node->offset;
+ nd->n = node->n;
+
+ /* the current nodes should become the new nodes */
+ node->nodes = new_nodes;
+ node->offset = k / BLOCKSZ;
+ node->n = 1;
+
+ if (!*key)
+ {
+ node->size = 1;
+ /* end of our key, store data in this node */
+ node->data = data;
+ }
+ else
+ {
+ /* we have more, make sure data for this node is NULL and
+ * add rest of our key to the nodes.
+ */
+ k = (uint8_t) *key;
+
+ if (CT_node_resize(node, k / BLOCKSZ))
+ {
+ return CT_ERR;
+ }
+ key++;
+ nd = CT_node_new(key, strlen(key), data);
+ if (nd == NULL)
+ {
+ return CT_ERR;
+ }
+
+ node->size = 2;
+ node->data = NULL;
+ (*node->nodes)[k - node->offset * BLOCKSZ] = nd;
+ }
+
+ /* re-allocate the key to free some space */
+ if ((new_sz = pt - node->key))
+ {
+ char * tmp = (char *) realloc(node->key, new_sz);
+ if (tmp != NULL)
+ {
+ node->key = tmp;
+ }
+ }
+ else
+ {
+ free(node->key);
+ node->key = NULL;
+ }
+ node->len = new_sz;
+
+ return CT_OK;
+ }
+ }
+
+ if (*key)
+ {
+ uint8_t k = (uint8_t) *key;
+
+ if (node->nodes == NULL)
+ {
+ if (CT_node_resize(node, k / BLOCKSZ))
+ {
+ return CT_ERR;
+ }
+ key++;
+ ct_node_t * nd = CT_node_new(key, strlen(key), data);
+ if (nd == NULL)
+ {
+ return CT_ERR;
+ }
+
+ node->size = 1;
+ (*node->nodes)[k - node->offset * BLOCKSZ ] = nd;
+
+ return CT_OK;
+ }
+
+ if (CT_node_resize(node, k / BLOCKSZ))
+ {
+ return CT_ERR;
+ }
+
+ ct_node_t ** nd = &(*node->nodes)[k - node->offset * BLOCKSZ];
+ key++;
+
+ if (*nd != NULL)
+ {
+ return CT_add(*nd, key, data);
+ }
+
+ *nd = CT_node_new(key, strlen(key), data);
+ if (*nd == NULL)
+ {
+ return CT_ERR;
+ }
+ node->size++;
+ return CT_OK;
+ }
+
+ if (node->data != NULL)
+ {
+ /* duplicate key */
+ return CT_EXISTS;
+ }
+ node->data = data;
+
+ return CT_OK;
+}
+
+/*
+ * Merge a child node with its parent for cleanup.
+ *
+ * This function should only be called when exactly one node is left and the
+ * node itself has no data.
+ *
+ * In case re-allocation fails the tree remains unchanged and therefore
+ * can still be used.
+ */
+static void CT_merge_node(ct_node_t * node)
+{
+ assert(node->size == 1 && node->data == NULL);
+ ct_node_t * child_node;
+ char * tmp;
+ uint_fast16_t i, end;
+
+ for (i = 0, end = node->n * BLOCKSZ; i < end; i++)
+ {
+ if ((*node->nodes)[i] != NULL)
+ {
+ break;
+ }
+ }
+ /* this is the child node we need to merge */
+ child_node = (*node->nodes)[i];
+
+ /* re-allocate enough space for the key + child_key + 1 char */
+ tmp = (char *) realloc(node->key, node->len + child_node->len + 1);
+ if (tmp == NULL)
+ {
+ log_error("Re-allocation failed while merging nodes in a c-tree");
+ return;
+ }
+ node->key = tmp;
+
+ /* set node char */
+ node->key[node->len++] = (char) (i + node->offset * BLOCKSZ);
+
+ /* append rest of the child key */
+ memcpy(node->key + node->len, child_node->key, child_node->len);
+ node->len += child_node->len;
+
+ /* free nodes (has only the child node left so nothing else
+ * needs cleaning */
+ free(node->nodes);
+
+ /* bind child nodes properties to the current node */
+ node->nodes = child_node->nodes;
+ node->size = child_node->size;
+ node->offset = child_node->offset;
+ node->n = child_node->n;
+ node->data = child_node->data;
+
+ /* free child key */
+ free(child_node->key);
+
+ /* free child node */
+ free(child_node);
+}
+
+/*
+ * This function can fail but in that case the tree is still usable.
+ */
+static void CT_dec_node(ct_node_t * node)
+{
+ if (node == NULL)
+ {
+ /* this is the root node */
+ return;
+ }
+
+ node->size--;
+
+ if (node->size == 0)
+ {
+ /* we can free nodes since they are no longer used */
+ free(node->nodes);
+
+ /* make sure to set nodes to NULL */
+ node->nodes = NULL;
+ }
+ else if (node->size == 1 && node->data == NULL)
+ {
+ CT_merge_node(node);
+ }
+}
+
+/*
+ * Removes and returns an item from the tree or NULL when not found.
+ */
+static void * CT_pop(ct_node_t * parent, ct_node_t ** nd, const char * key)
+{
+ ct_node_t * node = *nd;
+ if (strncmp(node->key, key, node->len))
+ {
+ return NULL;
+ }
+
+ key += node->len;
+
+ if (!*key)
+ {
+ void * data = node->data;
+ if (node->size == 0)
+ {
+ /* no child nodes, lets clean up this node */
+ CT_free(node, NULL);
+
+ /* make sure to set the node to NULL so the parent
+ * can do its cleanup correctly */
+ *nd = NULL;
+
+ /* size of parent should be minus one */
+ CT_dec_node(parent);
+
+ return data;
+ }
+ /* we cannot clean this node, set data to NULL */
+ node->data = NULL;
+
+ if (node->size == 1)
+ {
+ /* we have only one child, we can merge this
+ * child with this one */
+ CT_merge_node(node);
+ }
+
+ return data;
+ }
+
+ if (node->nodes != NULL)
+ {
+ uint8_t k = (uint8_t) *key;
+ uint8_t pos = k / BLOCKSZ;
+
+ if (pos < node->offset || pos >= node->offset + node->n)
+ {
+ return NULL;
+ }
+
+ ct_node_t ** next = &(*node->nodes)[k - node->offset * BLOCKSZ];
+
+ return (*next == NULL) ? NULL : CT_pop(node, next, key + 1);
+ }
+ return NULL;
+}
+
+/*
+ * Returns NULL in case an error has occurred.
+ */
+static ct_node_t * CT_node_new(const char * key, size_t len, void * data)
+{
+ ct_node_t * node = malloc(sizeof(ct_node_t));
+ if (node == NULL)
+ {
+ return NULL;
+ }
+ node->len = len;
+ node->data = data;
+ node->size = 0;
+ node->offset = UINT8_MAX;
+ node->n = 0;
+ node->nodes = NULL;
+ if (len)
+ {
+ node->key = malloc(len);
+ if (node->key == NULL)
+ {
+ free(node);
+ return NULL;
+ }
+ memcpy(node->key, key, len);
+ }
+ else
+ {
+ node->key = NULL;
+ }
+ return node;
+}
+
+/*
+ * Returns 0 is successful or -1 in case of an error.
+ *
+ * In case of an error, 'ct' remains unchanged.
+ */
+static int CT_node_resize(ct_node_t * node, uint8_t pos)
+{
+ int rc = 0;
+
+ if (node->nodes == NULL)
+ {
+ node->nodes = (ct_nodes_t *) calloc(1, sizeof(ct_nodes_t));
+ if (node->nodes == NULL)
+ {
+ rc = -1;
+ }
+ else
+ {
+ node->offset = pos;
+ node->n = 1;
+ }
+ }
+ else if (pos < node->offset)
+ {
+ ct_nodes_t * tmp;
+ uint8_t diff = node->offset - pos;
+ uint8_t oldn = node->n;
+ node->n += diff;
+ tmp = (ct_nodes_t *) realloc(
+ node->nodes,
+ node->n * sizeof(ct_nodes_t));
+ if (tmp == NULL && node->n)
+ {
+ node->n -= diff;
+ rc = -1;
+ }
+ else
+ {
+ node->nodes = tmp;
+ node->offset = pos;
+ memmove(node->nodes + diff,
+ node->nodes,
+ oldn * sizeof(ct_nodes_t));
+ memset(node->nodes, 0, diff * sizeof(ct_nodes_t));
+ }
+ }
+ else if (pos >= node->offset + node->n)
+ {
+ ct_nodes_t * tmp;
+ uint8_t diff = pos - node->offset - node->n + 1;
+ uint8_t oldn = node->n;
+ node->n += diff; /* assert node->n > 0 */
+ tmp = (ct_nodes_t *) realloc(
+ node->nodes,
+ node->n * sizeof(ct_nodes_t));
+ if (tmp == NULL)
+ {
+ node->n -= diff;
+ rc = -1;
+ }
+ else
+ {
+ node->nodes = tmp;
+ memset(node->nodes + oldn, 0, diff * sizeof(ct_nodes_t));
+ }
+ }
+
+ return rc;
+}
+
+/*
+ * Destroy ct_tree. (parsing NULL is NOT allowed)
+ * Call-back function will be called on each item in the tree.
+ */
+static void CT_free(ct_node_t * node, ct_free_cb cb)
+{
+ if (node->nodes != NULL)
+ {
+ uint_fast16_t i, end;
+ for (i = 0, end = node->n * BLOCKSZ; i < end; i++)
+ {
+ if ((*node->nodes)[i] != NULL)
+ {
+ CT_free((*node->nodes)[i], cb);
+ }
+ }
+ free(node->nodes);
+ }
+ if (cb != NULL && node->data != NULL)
+ {
+ (*cb)(node->data);
+ }
+ free(node->key);
+ free(node);
+}
+
--- /dev/null
+/*
+ * expr.c - For parsing expressions.
+ */
+#include <stdio.h>
+#include <ctype.h>
+#include <expr/expr.h>
+#include <stdlib.h>
+
+static int64_t expr_expression(const char ** expression);
+static int64_t expr_number(const char ** expression);
+static int64_t expr_factor(const char ** expression);
+static int64_t expr_term(const char ** expression);
+static int64_t expr_expression(const char ** expression);
+
+static int expr_err = 0;
+
+int expr_parse(int64_t * result, const char * expr)
+{
+ expr_err = 0;
+ *result = expr_expression(&expr);
+ return (expr_err) ? expr_err : 0;
+}
+
+static int64_t expr_number(const char ** expression)
+{
+ int64_t result = *(*expression)++ - '0';
+ while (**expression >= '0' && **expression <= '9')
+ {
+ result = 10 * result + *(*expression)++ - '0';
+ }
+
+ return result;
+}
+
+static int64_t expr_factor(const char ** expression)
+{
+ if (**expression >= '0' && **expression <= '9')
+ {
+ return expr_number(expression);
+ }
+ else if (**expression == '(')
+ {
+ (*expression)++; /* '(' */
+ int64_t result = expr_expression(expression);
+ (*expression)++; /* ')' */
+ return result;
+ }
+ else if (**expression == '-')
+ {
+ (*expression)++;
+ return -expr_factor(expression);
+ }
+ return 0; /* error */
+}
+
+static int64_t expr_term(const char ** expression)
+{
+ int64_t result = expr_factor(expression);
+ int64_t temp;
+ int i;
+
+ for (i = 1; i;)
+ switch (*(*expression))
+ {
+ case '*':
+ (*expression)++;
+ result *= expr_factor(expression);
+ break;
+ case '%':
+ (*expression)++;
+ result %= ((temp = expr_factor(expression))) ?
+ temp : (expr_err = EXPR_MODULO_BY_ZERO);
+ break;
+ case '/':
+ (*expression)++;
+ result /= ((temp = expr_factor(expression))) ?
+ temp : (expr_err = EXPR_DIVISION_BY_ZERO);
+ break;
+ default:
+ i = 0;
+ }
+ return result;
+}
+
+static int64_t expr_expression(const char ** expression)
+{
+ int64_t result = expr_term(expression);
+ while (**expression == '+' || **expression == '-')
+ {
+ if (*(*expression)++ == '+')
+ {
+ result += expr_term(expression);
+ }
+ else
+ {
+ result -= expr_term(expression);
+ }
+ }
+ return result;
+}
--- /dev/null
+/*
+ * imap.c - Lookup map for uint64_t integer keys with set operation support.
+ */
+#include <assert.h>
+#include <imap/imap.h>
+#include <logger/logger.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define IMAP_NODE_SZ 32
+
+static void IMAP_node_free(imap_node_t * node);
+static void IMAP_node_free_cb(imap_node_t * node, imap_free_cb cb);
+static int IMAP_set(imap_node_t * node, uint64_t id, void * data);
+static int IMAP_add(imap_node_t * node, uint64_t id, void * data);
+static void * IMAP_pop(imap_node_t * node, uint64_t id);
+static void IMAP_walk(imap_node_t * node, imap_cb cb, void * data, int * rc);
+static void IMAP_walkn(imap_node_t * node, imap_cb cb, void * data, size_t * n);
+static void IMAP_2vec(imap_node_t * node, vec_t * vec);
+static void IMAP_2vec_ref(imap_node_t * node, vec_t * vec);
+static void IMAP_union_ref(imap_node_t * dest, imap_node_t * node);
+static void IMAP_intersection_ref(
+ imap_node_t * dest,
+ imap_node_t * node,
+ imap_free_cb decref_cb);
+static void IMAP_difference_ref(
+ imap_node_t * dest,
+ imap_node_t * node,
+ imap_free_cb decref_cb);
+static void IMAP_symmetric_difference_ref(
+ imap_node_t * dest,
+ imap_node_t * node,
+ imap_free_cb decref_cb);
+
+/*
+ * Returns NULL in case an error has occurred.
+ */
+imap_t * imap_new(void)
+{
+ imap_t * imap = (imap_t *) calloc(
+ 1,
+ sizeof(imap_t) + IMAP_NODE_SZ * sizeof(imap_node_t));
+ if (imap == NULL)
+ {
+ return NULL;
+ }
+
+ imap->len = 0;
+ imap->vec = NULL;
+
+ return imap;
+}
+
+/*
+ * Destroy imap with optional call-back function.
+ */
+void imap_free(imap_t * imap, imap_free_cb cb)
+{
+ if (imap->len)
+ {
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ if (cb == NULL)
+ {
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = imap->nodes + i;
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_node_free(nd);
+ }
+ }
+ }
+ else
+ {
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = imap->nodes + i;
+
+ if (nd->data != NULL)
+ {
+ (*cb)(nd->data);
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_node_free_cb(nd, cb);
+ }
+ }
+ }
+ }
+
+ vec_free(imap->vec);
+ free(imap);
+}
+
+/*
+ * Add data by id to the map.
+ *
+ * Returns 0 when data is overwritten and 1 if a new id/value is set.
+ *
+ * In case of an error we return -1.
+ */
+int imap_set(imap_t * imap, uint64_t id, void * data)
+{
+ /* insert NULL is not allowed */
+ assert (data != NULL);
+ int rc;
+ imap_node_t * nd = imap->nodes + (id % IMAP_NODE_SZ);
+ id /= IMAP_NODE_SZ;
+
+ if (!id)
+ {
+ rc = (nd->data == NULL);
+
+ imap->len += rc;
+ nd->data = data;
+ }
+ else
+ {
+ rc = IMAP_set(nd, id - 1, data);
+
+ if (rc > 0)
+ {
+ imap->len++;
+ }
+ }
+
+ if (imap->vec != NULL && (
+ rc < 1 || vec_append_safe(&imap->vec, data)))
+ {
+ vec_free(imap->vec);
+ imap->vec = NULL;
+ }
+
+ return rc;
+}
+
+/*
+ * Add data by id to the map.
+ *
+ * Returns 0 when data is added. Data will NOT be overwritten.
+ *
+ * In case of a memory error we return -1. When the id already exists -2 will
+ * be returned.
+ */
+int imap_add(imap_t * imap, uint64_t id, void * data)
+{
+ /* insert NULL is not allowed */
+ assert (data != NULL);
+
+ imap_node_t * nd = imap->nodes + (id % IMAP_NODE_SZ);
+ id /= IMAP_NODE_SZ;
+
+ if (!id)
+ {
+ if (nd->data != NULL)
+ {
+ return -2;
+ }
+
+ imap->len++;
+ nd->data = data;
+ }
+ else
+ {
+ int rc = IMAP_add(nd, id - 1, data);
+ if (rc)
+ {
+ return rc;
+ }
+ imap->len++;
+ }
+
+ if (imap->vec != NULL && vec_append_safe(&imap->vec, data))
+ {
+ vec_free(imap->vec);
+ imap->vec = NULL;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns data by a given id, or NULL when not found.
+ */
+void * imap_get(imap_t * imap, uint64_t id)
+{
+ imap_node_t * nd = imap->nodes + (id % IMAP_NODE_SZ);
+ while (1)
+ {
+ id /= IMAP_NODE_SZ;
+
+ if (!id) return nd->data;
+ if (!nd->nodes) return NULL;
+
+ nd = nd->nodes + (--id % IMAP_NODE_SZ);
+ }
+}
+
+/*
+ * Remove and return an item by id or return NULL in case the id is not found.
+ */
+void * imap_pop(imap_t * imap, uint64_t id)
+{
+ void * data;
+ imap_node_t * nd = imap->nodes + (id % IMAP_NODE_SZ);
+ id /= IMAP_NODE_SZ;
+
+ if (id)
+ {
+ data = (nd->nodes == NULL) ? NULL : IMAP_pop(nd, id - 1);
+ }
+ else if ((data = nd->data) != NULL)
+ {
+ nd->data = NULL;
+ }
+
+ if (data != NULL)
+ {
+ imap->len--;
+
+ if (imap->vec != NULL)
+ {
+ vec_free(imap->vec);
+ imap->vec = NULL;
+ }
+ }
+
+ return data;
+}
+
+/*
+ * Run the call-back function on all items in the map.
+ *
+ * All the results are added together and are returned as the result of
+ * this function.
+ */
+int imap_walk(imap_t * imap, imap_cb cb, void * data)
+{
+ int rc = 0;
+
+ if (imap->len)
+ {
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = imap->nodes + i;
+
+ if (nd->data != NULL)
+ {
+ rc += (*cb)(nd->data, data);
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_walk(nd, cb, data, &rc);
+ }
+ }
+ }
+
+ return rc;
+}
+
+/*
+ * Recursive function, call-back function will be called on each item.
+ *
+ * Walking stops either when the call-back is called on each value or
+ * when 'n' is zero. 'n' will be decremented by the result of each call-back.
+ */
+void imap_walkn(imap_t * imap, size_t * n, imap_cb cb, void * data)
+{
+ if (imap->len)
+ {
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; *n && i < IMAP_NODE_SZ; i++)
+ {
+ nd = imap->nodes + i;
+
+ if (nd->data != NULL && !(*n -= (*cb)(nd->data, data)))
+ {
+ return;
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_walkn(nd, cb, data, n);
+ }
+ }
+ }
+}
+
+/*
+ * Returns NULL in case an error has occurred.
+ *
+ * When successful a BORROWED pointer to vec is returned.
+ */
+vec_t * imap_vec(imap_t * imap)
+{
+ if (imap->vec == NULL)
+ {
+ imap->vec = vec_new(imap->len);
+
+ if (imap->vec != NULL && imap->len)
+ {
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = imap->nodes + i;
+
+ if (nd->data != NULL)
+ {
+ vec_append(imap->vec, nd->data);
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_2vec(nd, imap->vec);
+ }
+ }
+ }
+ }
+ return imap->vec;
+}
+
+/*
+ * Returns NULL in case an error has occurred.
+ *
+ * When successful a the vec is returned and imap->vec is set to NULL.
+ *
+ * This can be used when being sure this is the only time you need the list
+ * and prevents making a copy.
+ */
+vec_t * imap_vec_pop(imap_t * imap)
+{
+ vec_t * vec = imap_vec(imap);
+ imap->vec = NULL;
+ return vec;
+}
+
+/*
+ * Returns NULL in case an error has occurred.
+ *
+ * When successful a NEW vec is returned.
+ */
+vec_t * imap_2vec(imap_t * imap)
+{
+ vec_t * vec = imap_vec(imap);
+ if (vec != NULL)
+ {
+ vec = vec_copy(vec);
+ }
+ return vec;
+}
+
+/*
+ * Use this function to create a s-list copy and update the ref count
+ * for each object. We expect each object to have object->ref (uint_xxx_t) on
+ * top of the object definition.
+ *
+ * There is no function to handle the decrement for the ref count since they
+ * are different for each object. Best is to handle the decrement while looping
+ * over the returned list.
+ *
+ * Returns NULL in case an error has occurred.
+ */
+vec_t * imap_2vec_ref(imap_t * imap)
+{
+ if (imap->vec == NULL)
+ {
+ imap->vec = vec_new(imap->len);
+
+ if (imap->vec != NULL && imap->len)
+ {
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = imap->nodes + i;
+
+ if (nd->data != NULL)
+ {
+ vec_append(imap->vec, nd->data);
+ vec_object_incref(nd->data);
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_2vec_ref(nd, imap->vec);
+ }
+ }
+ }
+ }
+ else
+ {
+ size_t i;
+ for (i = 0; i < imap->vec->len; i++)
+ {
+ vec_object_incref(imap->vec->data[i]);
+ }
+ }
+
+ return (imap->vec == NULL) ? NULL : vec_copy(imap->vec);
+}
+
+/*
+ * Map 'dest' will be the union between the two maps. Map 'imap' will be
+ * destroyed so it cannot be used anymore.
+ *
+ * This function can call 'decref_cb' when an item is removed from the map.
+ * We only call the function for sure when the item is removed from both maps.
+ * When we are sure the item still exists in the 'dest' map and is only removed
+ * from the 'imap', we simply decrement the ref counter.
+ */
+void imap_union_ref(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb __attribute__((unused)))
+{
+ if (dest->vec != NULL)
+ {
+ vec_free(dest->vec);
+ dest->vec = NULL;
+ }
+
+ if (imap->len)
+ {
+ imap_node_t * dest_nd;
+ imap_node_t * imap_nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ dest_nd = dest->nodes + i;
+ imap_nd = imap->nodes + i;
+
+ if (imap_nd->data != NULL)
+ {
+ if (dest_nd->data != NULL)
+ {
+ /* this must be the same object */
+ assert (imap_nd->data == dest_nd->data);
+ /* we are sure there is a ref left */
+ vec_object_decref(imap_nd->data);
+ }
+ else
+ {
+ dest_nd->data = imap_nd->data;
+ dest->len++;
+ }
+ }
+
+ if (imap_nd->nodes != NULL)
+ {
+ if (dest_nd->nodes != NULL)
+ {
+ size_t tmp = dest_nd->size;
+ IMAP_union_ref(dest_nd, imap_nd);
+ dest->len += dest_nd->size - tmp;
+ }
+ else
+ {
+ dest_nd->nodes = imap_nd->nodes;
+ dest_nd->size = imap_nd->size;
+ dest->len += dest_nd->size;
+ }
+ }
+ }
+ }
+
+ /* cleanup source imap */
+ vec_free(imap->vec);
+ free(imap);
+}
+
+/*
+ * Map 'dest' will be the intersection between the two maps. Map 'imap' will be
+ * destroyed so it cannot be used anymore.
+ *
+ * This function can call 'decref_cb' when an item is removed from the map.
+ * We only call the function for sure when the item is removed from both maps.
+ * When we are sure the item still exists in the 'dest' map and is only removed
+ * from the 'imap', we simply decrement the ref counter.
+ */
+void imap_intersection_ref(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb)
+{
+ if (dest->vec != NULL)
+ {
+ vec_free(dest->vec);
+ dest->vec = NULL;
+ }
+
+ imap_node_t * dest_nd;
+ imap_node_t * imap_nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ dest_nd = dest->nodes + i;
+ imap_nd = imap->nodes + i;
+ if (imap_nd->data != NULL)
+ {
+ (*decref_cb)(imap_nd->data);
+ }
+ else if (dest_nd->data != NULL)
+ {
+ (*decref_cb)(dest_nd->data);
+ dest_nd->data = NULL;
+ dest->len--;
+ }
+
+ if (imap_nd->nodes != NULL)
+ {
+ if (dest_nd->nodes != NULL)
+ {
+ size_t tmp = dest_nd->size;
+ IMAP_intersection_ref(dest_nd, imap_nd, decref_cb);
+ dest->len -= tmp - dest_nd->size;
+ }
+ else
+ {
+ IMAP_node_free_cb(imap_nd, decref_cb);
+ }
+ }
+ else if (dest_nd->nodes != NULL)
+ {
+ dest->len -= dest_nd->size;
+ IMAP_node_free_cb(dest_nd, decref_cb);
+ dest_nd->nodes = NULL;
+ }
+ }
+
+ /* cleanup source imap */
+ vec_free(imap->vec);
+ free(imap);
+}
+
+/*
+ * Map 'dest' will be the difference between the two maps. Map 'imap' will be
+ * destroyed so it cannot be used anymore.
+ *
+ * This function can call 'decref_cb' when an item is removed from the map.
+ * We only call the function for sure when the item is removed from both maps.
+ * When we are sure the item still exists in the 'dest' map and is only removed
+ * from the 'imap', we simply decrement the ref counter.
+ */
+void imap_difference_ref(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb)
+{
+ if (dest->vec != NULL)
+ {
+ vec_free(dest->vec);
+ dest->vec = NULL;
+ }
+
+ if (imap->len)
+ {
+ imap_node_t * dest_nd;
+ imap_node_t * imap_nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ dest_nd = dest->nodes + i;
+ imap_nd = imap->nodes + i;
+
+ if (imap_nd->data != NULL)
+ {
+ if (dest_nd->data != NULL)
+ {
+ /* this must be the same object */
+ assert (imap_nd->data == dest_nd->data);
+
+ /* we are sure to have one ref left */
+ vec_object_decref(dest_nd->data);
+ dest_nd->data = NULL;
+ dest->len--;
+
+ }
+ /* now we are not sure anymore if we have ref left */
+ (*decref_cb)(imap_nd->data);
+ }
+
+ if (imap_nd->nodes != NULL)
+ {
+ if (dest_nd->nodes != NULL)
+ {
+ size_t tmp = dest_nd->size;
+ IMAP_difference_ref(dest_nd, imap_nd, decref_cb);
+ dest->len -= tmp - dest_nd->size;
+ }
+ else
+ {
+ IMAP_node_free_cb(imap_nd, decref_cb);
+ }
+ }
+ }
+ }
+
+ /* cleanup source imap */
+ vec_free(imap->vec);
+ free(imap);
+}
+
+/*
+ * Map 'dest' will be the symmetric difference between the two maps. Map 'imap'
+ * will be destroyed so it cannot be used anymore.
+ *
+ * This function can call 'decref_cb' when an item is removed from the map.
+ * We only call the function for sure when the item is removed from both maps.
+ * When we are sure the item still exists in the 'dest' map and is only removed
+ * from the 'imap', we simply decrement the ref counter.
+ */
+void imap_symmetric_difference_ref(
+ imap_t * dest,
+ imap_t * imap,
+ imap_free_cb decref_cb)
+{
+ if (dest->vec != NULL)
+ {
+ vec_free(dest->vec);
+ dest->vec = NULL;
+ }
+
+ if (imap->len)
+ {
+ imap_node_t * dest_nd;
+ imap_node_t * imap_nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ dest_nd = dest->nodes + i;
+ imap_nd = imap->nodes + i;
+
+ if (imap_nd->data != NULL)
+ {
+ if (dest_nd->data != NULL)
+ {
+ /* this must be the same object */
+ assert (imap_nd->data == dest_nd->data);
+
+ /* we are sure to have one ref left */
+ vec_object_decref(dest_nd->data);
+
+ /* but now we are not sure anymore */
+ (*decref_cb)(imap_nd->data);
+
+ dest_nd->data = NULL;
+ dest->len--;
+ }
+ else
+ {
+ dest_nd->data = imap_nd->data;
+ dest->len++;
+ }
+ }
+
+ if (imap_nd->nodes != NULL)
+ {
+ if (dest_nd->nodes != NULL)
+ {
+ size_t tmp = dest_nd->size;
+ IMAP_symmetric_difference_ref(
+ dest_nd,
+ imap_nd,
+ decref_cb);
+ dest->len += dest_nd->size - tmp;
+ }
+ else
+ {
+ dest_nd->nodes = imap_nd->nodes;
+ dest_nd->size = imap_nd->size;
+ dest->len += dest_nd->size;
+ }
+ }
+ }
+ }
+
+ /* cleanup source imap */
+ vec_free(imap->vec);
+ free(imap);
+}
+
+static void IMAP_node_free(imap_node_t * node)
+{
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ if ((nd = node->nodes + i)->nodes != NULL)
+ {
+ IMAP_node_free(nd);
+ }
+ }
+
+ free(node->nodes);
+}
+
+static void IMAP_node_free_cb(imap_node_t * node, imap_free_cb cb)
+{
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = node->nodes + i;
+
+ if (nd->data != NULL)
+ {
+ (*cb)(nd->data);
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_node_free_cb(nd, cb);
+ }
+ }
+ free(node->nodes);
+}
+
+/*
+ * Add data by id to the map.
+ *
+ * Returns 0 when data is overwritten and 1 if a new id/value is set.
+ *
+ * In case of an error we return -1.
+ */
+static int IMAP_set(imap_node_t * node, uint64_t id, void * data)
+{
+ if (!node->size)
+ {
+ node->nodes = (imap_node_t *) calloc(
+ IMAP_NODE_SZ,
+ sizeof(imap_node_t));
+
+ if (node->nodes == NULL)
+ {
+ return -1;
+ }
+ }
+
+ int rc;
+ imap_node_t * nd = node->nodes + (id % IMAP_NODE_SZ);
+ id /= IMAP_NODE_SZ;
+
+ if (!id)
+ {
+ rc = (nd->data == NULL);
+
+ nd->data = data;
+ node->size += rc;
+
+ return rc;
+ }
+
+ rc = IMAP_set(nd, id - 1, data);
+
+ if (rc > 0)
+ {
+ node->size++;
+ }
+
+ return rc;
+}
+
+/*
+ * Add data by id to the map.
+ *
+ * Returns 0 when data is added. Data will NOT be overwritten.
+ *
+ * In case of a memory error we return -1. If the id
+ * already exists -2 will be returned.
+ */
+static int IMAP_add(imap_node_t * node, uint64_t id, void * data)
+{
+ if (!node->size)
+ {
+ node->nodes = (imap_node_t *) calloc(
+ IMAP_NODE_SZ,
+ sizeof(imap_node_t));
+
+ if (node->nodes == NULL)
+ {
+ return -1;
+ }
+ }
+
+ int rc;
+ imap_node_t * nd = node->nodes + (id % IMAP_NODE_SZ);
+ id /= IMAP_NODE_SZ;
+
+ if (!id)
+ {
+ if (nd->data != NULL)
+ {
+ return -2;
+ }
+
+ nd->data = data;
+ node->size++;
+
+ return 0;
+ }
+
+ rc = IMAP_add(nd, id - 1, data);
+
+ if (rc == 0)
+ {
+ node->size++;
+ }
+
+ return rc;
+}
+
+static void * IMAP_pop(imap_node_t * node, uint64_t id)
+{
+ void * data;
+ imap_node_t * nd = node->nodes + (id % IMAP_NODE_SZ);
+ id /= IMAP_NODE_SZ;
+
+ if (!id)
+ {
+ if ((data = nd->data) != NULL)
+ {
+ if (--node->size)
+ {
+ nd->data = NULL;
+ }
+ else
+ {
+ free(node->nodes);
+ node->nodes = NULL;
+ }
+ }
+
+ return data;
+ }
+
+ data = (nd->nodes == NULL) ? NULL : IMAP_pop(nd, id - 1);
+
+ if (data != NULL && !--node->size)
+ {
+ free(node->nodes);
+ node->nodes = NULL;
+ }
+
+ return data;
+}
+
+static void IMAP_walk(imap_node_t * node, imap_cb cb, void * data, int * rc)
+{
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = node->nodes + i;
+
+ if (nd->data != NULL)
+ {
+ *rc += (*cb)(nd->data, data);
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_walk(nd, cb, data, rc);
+ }
+ }
+}
+
+static void IMAP_walkn(imap_node_t * node, imap_cb cb, void * data, size_t * n)
+{
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; *n && i < IMAP_NODE_SZ; i++)
+ {
+ nd = node->nodes + i;
+
+ if (nd->data != NULL && !(*n -= (*cb)(nd->data, data)))
+ {
+ return;
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_walkn(nd, cb, data, n);
+ }
+ }
+}
+
+static void IMAP_2vec(imap_node_t * node, vec_t * vec)
+{
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = node->nodes + i;
+
+ if (nd->data != NULL)
+ {
+ vec_append(vec, nd->data);
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_2vec(nd, vec);
+ }
+ }
+}
+
+static void IMAP_2vec_ref(imap_node_t * node, vec_t * vec)
+{
+ imap_node_t * nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ nd = node->nodes + i;
+
+ if (nd->data != NULL)
+ {
+ vec_append(vec, nd->data);
+ vec_object_incref(nd->data);
+ }
+
+ if (nd->nodes != NULL)
+ {
+ IMAP_2vec_ref(nd, vec);
+ }
+ }
+}
+
+static void IMAP_union_ref(imap_node_t * dest, imap_node_t * node)
+{
+ imap_node_t * dest_nd;
+ imap_node_t * node_nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ dest_nd = dest->nodes + i;
+ node_nd = node->nodes + i;
+
+ if (node_nd->data != NULL)
+ {
+ if (dest_nd->data != NULL)
+ {
+ /* this must be the same object */
+ assert (node_nd->data == dest_nd->data);
+ /* we are sure there is a ref left */
+ vec_object_decref(node_nd->data);
+ }
+ else
+ {
+ dest_nd->data = node_nd->data;
+ dest->size++;
+ }
+ }
+
+ if (node_nd->nodes != NULL)
+ {
+ if (dest_nd->nodes != NULL)
+ {
+ size_t tmp = dest_nd->size;
+ IMAP_union_ref(dest_nd, node_nd);
+ dest->size += dest_nd->size - tmp;
+ }
+ else
+ {
+ dest_nd->nodes = node_nd->nodes;
+ dest_nd->size = node_nd->size;
+ dest->size += dest_nd->size;
+ }
+ }
+ }
+ free(node->nodes);
+}
+
+static void IMAP_intersection_ref(
+ imap_node_t * dest,
+ imap_node_t * node,
+ imap_free_cb decref_cb)
+{
+ imap_node_t * dest_nd;
+ imap_node_t * node_nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ dest_nd = dest->nodes + i;
+ node_nd = node->nodes + i;
+
+ if (node_nd->data != NULL)
+ {
+ (*decref_cb)(node_nd->data);
+ }
+ else if (dest_nd->data != NULL)
+ {
+ (*decref_cb)(dest_nd->data);
+ dest_nd->data = NULL;
+ dest->size--;
+ }
+
+ if (node_nd->nodes != NULL)
+ {
+ if (dest_nd->nodes != NULL)
+ {
+ size_t tmp = dest_nd->size;
+ IMAP_intersection_ref(dest_nd, node_nd, decref_cb);
+ dest->size -= tmp - dest_nd->size;
+ }
+ else
+ {
+ IMAP_node_free_cb(node_nd, decref_cb);
+ }
+ }
+ else if (dest_nd->nodes != NULL)
+ {
+ dest->size -= dest_nd->size;
+ IMAP_node_free_cb(dest_nd, decref_cb);
+ dest_nd->nodes = NULL;
+ }
+
+ }
+
+ if (!dest->size)
+ {
+ IMAP_node_free(dest);
+ dest->nodes = NULL;
+ }
+
+ free(node->nodes);
+}
+
+static void IMAP_difference_ref(
+ imap_node_t * dest,
+ imap_node_t * node,
+ imap_free_cb decref_cb)
+{
+ imap_node_t * dest_nd;
+ imap_node_t * node_nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ dest_nd = dest->nodes + i;
+ node_nd = node->nodes + i;
+
+ if (node_nd->data != NULL)
+ {
+ if (dest_nd->data != NULL)
+ {
+ /* this must be the same object */
+ assert (node_nd->data == dest_nd->data);
+ /* we are sure to have one ref left */
+ vec_object_decref(dest_nd->data);
+ dest_nd->data = NULL;
+ dest->size--;
+
+ }
+ /* now we are not sure anymore if we have ref left */
+ (*decref_cb)(node_nd->data);
+ }
+
+ if (node_nd->nodes != NULL)
+ {
+ if (dest_nd->nodes != NULL)
+ {
+ size_t tmp = dest_nd->size;
+ IMAP_difference_ref(dest_nd, node_nd, decref_cb);
+ dest->size -= tmp - dest_nd->size;
+ }
+ else
+ {
+ IMAP_node_free_cb(node_nd, decref_cb);
+ }
+ }
+ }
+
+ if (!dest->size)
+ {
+ IMAP_node_free(dest);
+ dest->nodes = NULL;
+ }
+
+ free(node->nodes);
+}
+
+static void IMAP_symmetric_difference_ref(
+ imap_node_t * dest,
+ imap_node_t * node,
+ imap_free_cb decref_cb)
+{
+ imap_node_t * dest_nd;
+ imap_node_t * node_nd;
+ uint_fast8_t i;
+
+ for (i = 0; i < IMAP_NODE_SZ; i++)
+ {
+ dest_nd = dest->nodes + i;
+ node_nd = node->nodes + i;
+
+ if (node_nd->data != NULL)
+ {
+ if (dest_nd->data != NULL)
+ {
+ /* this must be the same object */
+ assert (node_nd->data == dest_nd->data);
+
+ /* we are sure to have one ref left */
+ vec_object_decref(dest_nd->data);
+
+ /* but now we are not sure anymore */
+ (*decref_cb)(node_nd->data);
+
+ dest_nd->data = NULL;
+ dest->size--;
+ }
+ else
+ {
+ dest_nd->data = node_nd->data;
+ dest->size++;
+ }
+ }
+
+ if (node_nd->nodes != NULL)
+ {
+ if (dest_nd->nodes != NULL)
+ {
+ size_t tmp = dest_nd->size;
+ IMAP_symmetric_difference_ref(
+ dest_nd,
+ node_nd,
+ decref_cb);
+ dest->size += dest_nd->size - tmp;
+ }
+ else
+ {
+ dest_nd->nodes = node_nd->nodes;
+ dest_nd->size = node_nd->size;
+ dest->size += dest_nd->size;
+ }
+ }
+ }
+
+ if (!dest->size)
+ {
+ IMAP_node_free(dest);
+ dest->nodes = NULL;
+ }
+
+ free(node->nodes);
+}
--- /dev/null
+/*
+ * iso8601.c - Library to parse ISO 8601 dates. Time-zones are found with
+ * tzset() in either /usr/lib/zoneinfo/ or /usr/share/zoneinfo/.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <iso8601/iso8601.h>
+#include <stdlib.h>
+#include <time.h>
+#include <string.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <inttypes.h>
+#include <assert.h>
+
+static const char * tz_common[] = {
+ "TZ=:localtime",
+ "TZ=:Africa/Abidjan",
+ "TZ=:Africa/Accra",
+ "TZ=:Africa/Addis_Ababa",
+ "TZ=:Africa/Algiers",
+ "TZ=:Africa/Asmara",
+ "TZ=:Africa/Bamako",
+ "TZ=:Africa/Bangui",
+ "TZ=:Africa/Banjul",
+ "TZ=:Africa/Bissau",
+ "TZ=:Africa/Blantyre",
+ "TZ=:Africa/Brazzaville",
+ "TZ=:Africa/Bujumbura",
+ "TZ=:Africa/Cairo",
+ "TZ=:Africa/Casablanca",
+ "TZ=:Africa/Ceuta",
+ "TZ=:Africa/Conakry",
+ "TZ=:Africa/Dakar",
+ "TZ=:Africa/Dar_es_Salaam",
+ "TZ=:Africa/Djibouti",
+ "TZ=:Africa/Douala",
+ "TZ=:Africa/El_Aaiun",
+ "TZ=:Africa/Freetown",
+ "TZ=:Africa/Gaborone",
+ "TZ=:Africa/Harare",
+ "TZ=:Africa/Johannesburg",
+ "TZ=:Africa/Juba",
+ "TZ=:Africa/Kampala",
+ "TZ=:Africa/Khartoum",
+ "TZ=:Africa/Kigali",
+ "TZ=:Africa/Kinshasa",
+ "TZ=:Africa/Lagos",
+ "TZ=:Africa/Libreville",
+ "TZ=:Africa/Lome",
+ "TZ=:Africa/Luanda",
+ "TZ=:Africa/Lubumbashi",
+ "TZ=:Africa/Lusaka",
+ "TZ=:Africa/Malabo",
+ "TZ=:Africa/Maputo",
+ "TZ=:Africa/Maseru",
+ "TZ=:Africa/Mbabane",
+ "TZ=:Africa/Mogadishu",
+ "TZ=:Africa/Monrovia",
+ "TZ=:Africa/Nairobi",
+ "TZ=:Africa/Ndjamena",
+ "TZ=:Africa/Niamey",
+ "TZ=:Africa/Nouakchott",
+ "TZ=:Africa/Ouagadougou",
+ "TZ=:Africa/Porto-Novo",
+ "TZ=:Africa/Sao_Tome",
+ "TZ=:Africa/Tripoli",
+ "TZ=:Africa/Tunis",
+ "TZ=:Africa/Windhoek",
+ "TZ=:America/Adak",
+ "TZ=:America/Anchorage",
+ "TZ=:America/Anguilla",
+ "TZ=:America/Antigua",
+ "TZ=:America/Araguaina",
+ "TZ=:America/Argentina/Buenos_Aires",
+ "TZ=:America/Argentina/Catamarca",
+ "TZ=:America/Argentina/Cordoba",
+ "TZ=:America/Argentina/Jujuy",
+ "TZ=:America/Argentina/La_Rioja",
+ "TZ=:America/Argentina/Mendoza",
+ "TZ=:America/Argentina/Rio_Gallegos",
+ "TZ=:America/Argentina/Salta",
+ "TZ=:America/Argentina/San_Juan",
+ "TZ=:America/Argentina/San_Luis",
+ "TZ=:America/Argentina/Tucuman",
+ "TZ=:America/Argentina/Ushuaia",
+ "TZ=:America/Aruba",
+ "TZ=:America/Asuncion",
+ "TZ=:America/Atikokan",
+ "TZ=:America/Bahia",
+ "TZ=:America/Bahia_Banderas",
+ "TZ=:America/Barbados",
+ "TZ=:America/Belem",
+ "TZ=:America/Belize",
+ "TZ=:America/Blanc-Sablon",
+ "TZ=:America/Boa_Vista",
+ "TZ=:America/Bogota",
+ "TZ=:America/Boise",
+ "TZ=:America/Cambridge_Bay",
+ "TZ=:America/Campo_Grande",
+ "TZ=:America/Cancun",
+ "TZ=:America/Caracas",
+ "TZ=:America/Cayenne",
+ "TZ=:America/Cayman",
+ "TZ=:America/Chicago",
+ "TZ=:America/Chihuahua",
+ "TZ=:America/Costa_Rica",
+ "TZ=:America/Creston",
+ "TZ=:America/Cuiaba",
+ "TZ=:America/Curacao",
+ "TZ=:America/Danmarkshavn",
+ "TZ=:America/Dawson",
+ "TZ=:America/Dawson_Creek",
+ "TZ=:America/Denver",
+ "TZ=:America/Detroit",
+ "TZ=:America/Dominica",
+ "TZ=:America/Edmonton",
+ "TZ=:America/Eirunepe",
+ "TZ=:America/El_Salvador",
+ "TZ=:America/Fort_Nelson",
+ "TZ=:America/Fortaleza",
+ "TZ=:America/Glace_Bay",
+ "TZ=:America/Godthab",
+ "TZ=:America/Goose_Bay",
+ "TZ=:America/Grand_Turk",
+ "TZ=:America/Grenada",
+ "TZ=:America/Guadeloupe",
+ "TZ=:America/Guatemala",
+ "TZ=:America/Guayaquil",
+ "TZ=:America/Guyana",
+ "TZ=:America/Halifax",
+ "TZ=:America/Havana",
+ "TZ=:America/Hermosillo",
+ "TZ=:America/Indiana/Indianapolis",
+ "TZ=:America/Indiana/Knox",
+ "TZ=:America/Indiana/Marengo",
+ "TZ=:America/Indiana/Petersburg",
+ "TZ=:America/Indiana/Tell_City",
+ "TZ=:America/Indiana/Vevay",
+ "TZ=:America/Indiana/Vincennes",
+ "TZ=:America/Indiana/Winamac",
+ "TZ=:America/Inuvik",
+ "TZ=:America/Iqaluit",
+ "TZ=:America/Jamaica",
+ "TZ=:America/Juneau",
+ "TZ=:America/Kentucky/Louisville",
+ "TZ=:America/Kentucky/Monticello",
+ "TZ=:America/Kralendijk",
+ "TZ=:America/La_Paz",
+ "TZ=:America/Lima",
+ "TZ=:America/Los_Angeles",
+ "TZ=:America/Lower_Princes",
+ "TZ=:America/Maceio",
+ "TZ=:America/Managua",
+ "TZ=:America/Manaus",
+ "TZ=:America/Marigot",
+ "TZ=:America/Martinique",
+ "TZ=:America/Matamoros",
+ "TZ=:America/Mazatlan",
+ "TZ=:America/Menominee",
+ "TZ=:America/Merida",
+ "TZ=:America/Metlakatla",
+ "TZ=:America/Mexico_City",
+ "TZ=:America/Miquelon",
+ "TZ=:America/Moncton",
+ "TZ=:America/Monterrey",
+ "TZ=:America/Montevideo",
+ "TZ=:America/Montserrat",
+ "TZ=:America/Nassau",
+ "TZ=:America/New_York",
+ "TZ=:America/Nipigon",
+ "TZ=:America/Nome",
+ "TZ=:America/Noronha",
+ "TZ=:America/North_Dakota/Beulah",
+ "TZ=:America/North_Dakota/Center",
+ "TZ=:America/North_Dakota/New_Salem",
+ "TZ=:America/Ojinaga",
+ "TZ=:America/Panama",
+ "TZ=:America/Pangnirtung",
+ "TZ=:America/Paramaribo",
+ "TZ=:America/Phoenix",
+ "TZ=:America/Port-au-Prince",
+ "TZ=:America/Port_of_Spain",
+ "TZ=:America/Porto_Velho",
+ "TZ=:America/Puerto_Rico",
+ "TZ=:America/Rainy_River",
+ "TZ=:America/Rankin_Inlet",
+ "TZ=:America/Recife",
+ "TZ=:America/Regina",
+ "TZ=:America/Resolute",
+ "TZ=:America/Rio_Branco",
+ "TZ=:America/Santa_Isabel",
+ "TZ=:America/Santarem",
+ "TZ=:America/Santiago",
+ "TZ=:America/Santo_Domingo",
+ "TZ=:America/Sao_Paulo",
+ "TZ=:America/Scoresbysund",
+ "TZ=:America/Sitka",
+ "TZ=:America/St_Barthelemy",
+ "TZ=:America/St_Johns",
+ "TZ=:America/St_Kitts",
+ "TZ=:America/St_Lucia",
+ "TZ=:America/St_Thomas",
+ "TZ=:America/St_Vincent",
+ "TZ=:America/Swift_Current",
+ "TZ=:America/Tegucigalpa",
+ "TZ=:America/Thule",
+ "TZ=:America/Thunder_Bay",
+ "TZ=:America/Tijuana",
+ "TZ=:America/Toronto",
+ "TZ=:America/Tortola",
+ "TZ=:America/Vancouver",
+ "TZ=:America/Whitehorse",
+ "TZ=:America/Winnipeg",
+ "TZ=:America/Yakutat",
+ "TZ=:America/Yellowknife",
+ "TZ=:Antarctica/Casey",
+ "TZ=:Antarctica/Davis",
+ "TZ=:Antarctica/DumontDUrville",
+ "TZ=:Antarctica/Macquarie",
+ "TZ=:Antarctica/Mawson",
+ "TZ=:Antarctica/McMurdo",
+ "TZ=:Antarctica/Palmer",
+ "TZ=:Antarctica/Rothera",
+ "TZ=:Antarctica/Syowa",
+ "TZ=:Antarctica/Troll",
+ "TZ=:Antarctica/Vostok",
+ "TZ=:Arctic/Longyearbyen",
+ "TZ=:Asia/Aden",
+ "TZ=:Asia/Almaty",
+ "TZ=:Asia/Amman",
+ "TZ=:Asia/Anadyr",
+ "TZ=:Asia/Aqtau",
+ "TZ=:Asia/Aqtobe",
+ "TZ=:Asia/Ashgabat",
+ "TZ=:Asia/Baghdad",
+ "TZ=:Asia/Bahrain",
+ "TZ=:Asia/Baku",
+ "TZ=:Asia/Bangkok",
+ "TZ=:Asia/Beirut",
+ "TZ=:Asia/Bishkek",
+ "TZ=:Asia/Brunei",
+ "TZ=:Asia/Chita",
+ "TZ=:Asia/Choibalsan",
+ "TZ=:Asia/Colombo",
+ "TZ=:Asia/Damascus",
+ "TZ=:Asia/Dhaka",
+ "TZ=:Asia/Dili",
+ "TZ=:Asia/Dubai",
+ "TZ=:Asia/Dushanbe",
+ "TZ=:Asia/Gaza",
+ "TZ=:Asia/Hebron",
+ "TZ=:Asia/Ho_Chi_Minh",
+ "TZ=:Asia/Hong_Kong",
+ "TZ=:Asia/Hovd",
+ "TZ=:Asia/Irkutsk",
+ "TZ=:Asia/Jakarta",
+ "TZ=:Asia/Jayapura",
+ "TZ=:Asia/Jerusalem",
+ "TZ=:Asia/Kabul",
+ "TZ=:Asia/Kamchatka",
+ "TZ=:Asia/Karachi",
+ "TZ=:Asia/Kathmandu",
+ "TZ=:Asia/Khandyga",
+ "TZ=:Asia/Kolkata",
+ "TZ=:Asia/Krasnoyarsk",
+ "TZ=:Asia/Kuala_Lumpur",
+ "TZ=:Asia/Kuching",
+ "TZ=:Asia/Kuwait",
+ "TZ=:Asia/Macau",
+ "TZ=:Asia/Magadan",
+ "TZ=:Asia/Makassar",
+ "TZ=:Asia/Manila",
+ "TZ=:Asia/Muscat",
+ "TZ=:Asia/Nicosia",
+ "TZ=:Asia/Novokuznetsk",
+ "TZ=:Asia/Novosibirsk",
+ "TZ=:Asia/Omsk",
+ "TZ=:Asia/Oral",
+ "TZ=:Asia/Phnom_Penh",
+ "TZ=:Asia/Pontianak",
+ "TZ=:Asia/Pyongyang",
+ "TZ=:Asia/Qatar",
+ "TZ=:Asia/Qyzylorda",
+ "TZ=:Asia/Rangoon",
+ "TZ=:Asia/Riyadh",
+ "TZ=:Asia/Sakhalin",
+ "TZ=:Asia/Samarkand",
+ "TZ=:Asia/Seoul",
+ "TZ=:Asia/Shanghai",
+ "TZ=:Asia/Singapore",
+ "TZ=:Asia/Srednekolymsk",
+ "TZ=:Asia/Taipei",
+ "TZ=:Asia/Tashkent",
+ "TZ=:Asia/Tbilisi",
+ "TZ=:Asia/Tehran",
+ "TZ=:Asia/Thimphu",
+ "TZ=:Asia/Tokyo",
+ "TZ=:Asia/Ulaanbaatar",
+ "TZ=:Asia/Urumqi",
+ "TZ=:Asia/Ust-Nera",
+ "TZ=:Asia/Vientiane",
+ "TZ=:Asia/Vladivostok",
+ "TZ=:Asia/Yakutsk",
+ "TZ=:Asia/Yekaterinburg",
+ "TZ=:Asia/Yerevan",
+ "TZ=:Atlantic/Azores",
+ "TZ=:Atlantic/Bermuda",
+ "TZ=:Atlantic/Canary",
+ "TZ=:Atlantic/Cape_Verde",
+ "TZ=:Atlantic/Faroe",
+ "TZ=:Atlantic/Madeira",
+ "TZ=:Atlantic/Reykjavik",
+ "TZ=:Atlantic/South_Georgia",
+ "TZ=:Atlantic/St_Helena",
+ "TZ=:Atlantic/Stanley",
+ "TZ=:Australia/Adelaide",
+ "TZ=:Australia/Brisbane",
+ "TZ=:Australia/Broken_Hill",
+ "TZ=:Australia/Currie",
+ "TZ=:Australia/Darwin",
+ "TZ=:Australia/Eucla",
+ "TZ=:Australia/Hobart",
+ "TZ=:Australia/Lindeman",
+ "TZ=:Australia/Lord_Howe",
+ "TZ=:Australia/Melbourne",
+ "TZ=:Australia/Perth",
+ "TZ=:Australia/Sydney",
+ "TZ=:Canada/Atlantic",
+ "TZ=:Canada/Central",
+ "TZ=:Canada/Eastern",
+ "TZ=:Canada/Mountain",
+ "TZ=:Canada/Newfoundland",
+ "TZ=:Canada/Pacific",
+ "TZ=:Europe/Amsterdam",
+ "TZ=:Europe/Andorra",
+ "TZ=:Europe/Athens",
+ "TZ=:Europe/Belgrade",
+ "TZ=:Europe/Berlin",
+ "TZ=:Europe/Bratislava",
+ "TZ=:Europe/Brussels",
+ "TZ=:Europe/Bucharest",
+ "TZ=:Europe/Budapest",
+ "TZ=:Europe/Busingen",
+ "TZ=:Europe/Chisinau",
+ "TZ=:Europe/Copenhagen",
+ "TZ=:Europe/Dublin",
+ "TZ=:Europe/Gibraltar",
+ "TZ=:Europe/Guernsey",
+ "TZ=:Europe/Helsinki",
+ "TZ=:Europe/Isle_of_Man",
+ "TZ=:Europe/Istanbul",
+ "TZ=:Europe/Jersey",
+ "TZ=:Europe/Kaliningrad",
+ "TZ=:Europe/Kiev",
+ "TZ=:Europe/Lisbon",
+ "TZ=:Europe/Ljubljana",
+ "TZ=:Europe/London",
+ "TZ=:Europe/Luxembourg",
+ "TZ=:Europe/Madrid",
+ "TZ=:Europe/Malta",
+ "TZ=:Europe/Mariehamn",
+ "TZ=:Europe/Minsk",
+ "TZ=:Europe/Monaco",
+ "TZ=:Europe/Moscow",
+ "TZ=:Europe/Oslo",
+ "TZ=:Europe/Paris",
+ "TZ=:Europe/Podgorica",
+ "TZ=:Europe/Prague",
+ "TZ=:Europe/Riga",
+ "TZ=:Europe/Rome",
+ "TZ=:Europe/Samara",
+ "TZ=:Europe/San_Marino",
+ "TZ=:Europe/Sarajevo",
+ "TZ=:Europe/Simferopol",
+ "TZ=:Europe/Skopje",
+ "TZ=:Europe/Sofia",
+ "TZ=:Europe/Stockholm",
+ "TZ=:Europe/Tallinn",
+ "TZ=:Europe/Tirane",
+ "TZ=:Europe/Uzhgorod",
+ "TZ=:Europe/Vaduz",
+ "TZ=:Europe/Vatican",
+ "TZ=:Europe/Vienna",
+ "TZ=:Europe/Vilnius",
+ "TZ=:Europe/Volgograd",
+ "TZ=:Europe/Warsaw",
+ "TZ=:Europe/Zagreb",
+ "TZ=:Europe/Zaporozhye",
+ "TZ=:Europe/Zurich",
+ "TZ=:GMT",
+ "TZ=:Indian/Antananarivo",
+ "TZ=:Indian/Chagos",
+ "TZ=:Indian/Christmas",
+ "TZ=:Indian/Cocos",
+ "TZ=:Indian/Comoro",
+ "TZ=:Indian/Kerguelen",
+ "TZ=:Indian/Mahe",
+ "TZ=:Indian/Maldives",
+ "TZ=:Indian/Mauritius",
+ "TZ=:Indian/Mayotte",
+ "TZ=:Indian/Reunion",
+ "TZ=:Pacific/Apia",
+ "TZ=:Pacific/Auckland",
+ "TZ=:Pacific/Bougainville",
+ "TZ=:Pacific/Chatham",
+ "TZ=:Pacific/Chuuk",
+ "TZ=:Pacific/Easter",
+ "TZ=:Pacific/Efate",
+ "TZ=:Pacific/Enderbury",
+ "TZ=:Pacific/Fakaofo",
+ "TZ=:Pacific/Fiji",
+ "TZ=:Pacific/Funafuti",
+ "TZ=:Pacific/Galapagos",
+ "TZ=:Pacific/Gambier",
+ "TZ=:Pacific/Guadalcanal",
+ "TZ=:Pacific/Guam",
+ "TZ=:Pacific/Honolulu",
+ "TZ=:Pacific/Johnston",
+ "TZ=:Pacific/Kiritimati",
+ "TZ=:Pacific/Kosrae",
+ "TZ=:Pacific/Kwajalein",
+ "TZ=:Pacific/Majuro",
+ "TZ=:Pacific/Marquesas",
+ "TZ=:Pacific/Midway",
+ "TZ=:Pacific/Nauru",
+ "TZ=:Pacific/Niue",
+ "TZ=:Pacific/Norfolk",
+ "TZ=:Pacific/Noumea",
+ "TZ=:Pacific/Pago_Pago",
+ "TZ=:Pacific/Palau",
+ "TZ=:Pacific/Pitcairn",
+ "TZ=:Pacific/Pohnpei",
+ "TZ=:Pacific/Port_Moresby",
+ "TZ=:Pacific/Rarotonga",
+ "TZ=:Pacific/Saipan",
+ "TZ=:Pacific/Tahiti",
+ "TZ=:Pacific/Tarawa",
+ "TZ=:Pacific/Tongatapu",
+ "TZ=:Pacific/Wake",
+ "TZ=:Pacific/Wallis",
+ "TZ=:US/Alaska",
+ "TZ=:US/Arizona",
+ "TZ=:US/Central",
+ "TZ=:US/Eastern",
+ "TZ=:US/Hawaii",
+ "TZ=:US/Mountain",
+ "TZ=:US/Pacific",
+ "TZ=:UTC"
+};
+
+#define TZ_LEN 433
+#define TZ_UTC 432
+
+/* Use this offset because names start with TZ=: */
+#define TZ_NAME_OFFSET 4
+
+#define TZ_DIGITS \
+ for (len = 0; *pt && isdigit(*pt); pt++, len++);
+
+#define TZ_RESULT(tz_fmt, check_min) \
+ if (*pt == 0) \
+ return get_ts(str, tz_fmt, tz, 0); \
+ \
+ if (*pt == 'Z' && *(pt + 1) == 0) \
+ return get_ts(str, tz_fmt "Z", TZ_UTC, 0); \
+ \
+ if (*pt == '+' || (check_min && *pt == '-' && (sign = 1))) \
+ { \
+ offset = 0; \
+ \
+ pt++; \
+ if (!isdigit(*pt)) \
+ return -1; \
+ offset += (*pt - '0') * 36000; \
+ \
+ pt++; \
+ if (!isdigit(*pt)) \
+ return -1; \
+ offset += (*pt - '0') * 3600; \
+ \
+ pt++; \
+ if (*pt == ':') \
+ pt++; \
+ \
+ if (*pt == 0) \
+ return get_ts(str, tz_fmt, TZ_UTC, offset * sign); \
+ \
+ if (!isdigit(*pt)) \
+ return -1; \
+ offset += (*pt - '0') * 600; \
+ \
+ pt++; \
+ if (!isdigit(*pt)) \
+ return -1; \
+ offset += (*pt - '0') * 60; \
+ \
+ pt++; \
+ if (*pt != 0) \
+ return -1; \
+ \
+ return get_ts(str, tz_fmt, TZ_UTC, offset * sign); \
+ }
+
+
+static int64_t get_ts(
+ const char * str,
+ const char * fmt,
+ iso8601_tz_t tz,
+ int64_t offset);
+
+iso8601_tz_t iso8601_tz(const char * tzname)
+{
+ size_t i, j;
+ size_t len = strlen(tzname);
+ char buf[len];
+
+ for (i = 0; i < len; i++)
+ {
+ buf[i] = tolower(tzname[i]);
+ }
+
+ if (len == strlen("naive") && strncmp(buf, "naive", len) == 0)
+ {
+ /* TZ=:localtime */
+ return 0;
+ }
+
+ for (i = 0; i < TZ_LEN; i++)
+ {
+ if (strlen(tz_common[i]) != len + TZ_NAME_OFFSET)
+ {
+ continue;
+ }
+
+ for (j = 0; j < len; j++)
+ {
+ if (buf[j] != tolower(tz_common[i][j + TZ_NAME_OFFSET]))
+ {
+ break;
+ }
+ }
+
+ if (j != len)
+ {
+ continue;
+ }
+
+ return i;
+ }
+
+ return -1;
+}
+
+const char * iso8601_tzname(iso8601_tz_t tz)
+{
+ return (tz) ? tz_common[tz] + TZ_NAME_OFFSET : "NAIVE";
+}
+
+int64_t iso8601_parse_date(const char * str, iso8601_tz_t tz)
+{
+ size_t len;
+ const char * pt = str;
+ int64_t offset;
+ int sign = -1;
+ int use_t = 0;
+
+ /* read year */
+ TZ_DIGITS
+ if (len != 4)
+ {
+ return -1;
+ }
+
+ TZ_RESULT("%Y", 0)
+ if (*pt != '-')
+ {
+ return -1;
+ }
+ pt++;
+
+ /* read month */
+ TZ_DIGITS
+ if (!len || len > 2)
+ {
+ return -1;
+ }
+
+ TZ_RESULT("%Y-%m", 0)
+ if (*pt != '-')
+ {
+ return -1;
+ }
+ pt++;
+
+ /* read day */
+ TZ_DIGITS
+ if (!len || len > 2)
+ {
+ return -1;
+ }
+
+ TZ_RESULT("%Y-%m-%d", 1)
+ if (*pt != ' ' && (use_t = 1) && *pt != 'T')
+ {
+ return -1;
+ }
+ pt++;
+
+ /* read hours */
+ for (len = 0; *pt && isdigit(*pt); pt++, len++);
+
+ if (!len || len > 2)
+ {
+ return -1;
+ }
+
+ TZ_RESULT((use_t) ? "%Y-%m-%dT%H" : "%Y-%m-%d %H", 1)
+ if (*pt != ':')
+ {
+ return -1;
+ }
+ pt++;
+
+ /* read minutes */
+ TZ_DIGITS
+ if (!len || len > 2)
+ {
+ return -1;
+ }
+
+ TZ_RESULT((use_t) ? "%Y-%m-%dT%H:%M" : "%Y-%m-%d %H:%M", 1)
+ if (*pt != ':')
+ {
+ return -1;
+ }
+ pt++;
+
+ /* read seconds */
+ TZ_DIGITS
+ if (!len || len > 2)
+ {
+ return -1;
+ }
+
+ TZ_RESULT((use_t) ? "%Y-%m-%dT%H:%M:%S" : "%Y-%m-%d %H:%M:%S", 1)
+
+ return -1;
+}
+
+static int64_t get_ts(
+ const char * str,
+ const char * fmt,
+ iso8601_tz_t tz,
+ int64_t offset)
+{
+ struct tm tm;
+ time_t ts;
+
+ /* must be a valid timezone */
+ assert (tz >= 0 && tz < TZ_LEN);
+
+ /* timezone must be 0 (UTC) */
+ assert (timezone == 0);
+
+ memset(&tm, 0, sizeof(struct tm));
+
+ /* month day should default to 1, not 0 */
+ if (!tm.tm_mday)
+ {
+ tm.tm_mday = 1;
+ }
+
+ if (strptime(str, fmt, &tm) == NULL)
+ {
+ /* parsing date string failed */
+ return -1;
+ }
+
+ ts = mktime(&tm);
+
+ /* in case of UTO or a custom offset we can simple return */
+ if (tz == TZ_UTC || offset)
+ {
+ return ts + offset;
+ }
+
+ /* set custom timezone */
+ putenv((char *) tz_common[tz]);
+ tzset();
+
+ /* localize the timestamp so we get the correct offset including
+ * daylight saving time.
+ */
+ memset(&tm, 0, sizeof(struct tm));
+ localtime_r(&ts, &tm);
+
+ /* set environment back to UTC */
+ putenv("TZ=:UTC");
+ tzset();
+
+ ts -= tm.tm_gmtoff;
+
+ return ts;
+}
--- /dev/null
+/* Copyright Joyent, Inc. and other Node contributors.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#include <lib/http_parser.h>
+#include <assert.h>
+#include <stddef.h>
+#include <ctype.h>
+#include <string.h>
+#include <limits.h>
+
+static uint32_t max_header_size = HTTP_MAX_HEADER_SIZE;
+
+#ifndef ULLONG_MAX
+# define ULLONG_MAX ((uint64_t) -1) /* 2^64-1 */
+#endif
+
+#ifndef MIN
+# define MIN(a,b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef ARRAY_SIZE
+# define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
+#endif
+
+#ifndef BIT_AT
+# define BIT_AT(a, i) \
+ (!!((unsigned int) (a)[(unsigned int) (i) >> 3] & \
+ (1 << ((unsigned int) (i) & 7))))
+#endif
+
+#ifndef ELEM_AT
+# define ELEM_AT(a, i, v) ((unsigned int) (i) < ARRAY_SIZE(a) ? (a)[(i)] : (v))
+#endif
+
+#define SET_ERRNO(e) \
+do { \
+ parser->nread = nread; \
+ parser->http_errno = (e); \
+} while(0)
+
+#define CURRENT_STATE() p_state
+#define UPDATE_STATE(V) p_state = (enum state) (V);
+#define RETURN(V) \
+do { \
+ parser->nread = nread; \
+ parser->state = CURRENT_STATE(); \
+ return (V); \
+} while (0);
+#define REEXECUTE() \
+ goto reexecute; \
+
+
+#ifdef __GNUC__
+# define LIKELY(X) __builtin_expect(!!(X), 1)
+# define UNLIKELY(X) __builtin_expect(!!(X), 0)
+#else
+# define LIKELY(X) (X)
+# define UNLIKELY(X) (X)
+#endif
+
+
+/* Run the notify callback FOR, returning ER if it fails */
+#define CALLBACK_NOTIFY_(FOR, ER) \
+do { \
+ assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \
+ \
+ if (LIKELY(settings->on_##FOR)) { \
+ parser->state = CURRENT_STATE(); \
+ if (UNLIKELY(0 != settings->on_##FOR(parser))) { \
+ SET_ERRNO(HPE_CB_##FOR); \
+ } \
+ UPDATE_STATE(parser->state); \
+ \
+ /* We either errored above or got paused; get out */ \
+ if (UNLIKELY(HTTP_PARSER_ERRNO(parser) != HPE_OK)) { \
+ return (ER); \
+ } \
+ } \
+} while (0)
+
+/* Run the notify callback FOR and consume the current byte */
+#define CALLBACK_NOTIFY(FOR) CALLBACK_NOTIFY_(FOR, p - data + 1)
+
+/* Run the notify callback FOR and don't consume the current byte */
+#define CALLBACK_NOTIFY_NOADVANCE(FOR) CALLBACK_NOTIFY_(FOR, p - data)
+
+/* Run data callback FOR with LEN bytes, returning ER if it fails */
+#define CALLBACK_DATA_(FOR, LEN, ER) \
+do { \
+ assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \
+ \
+ if (FOR##_mark) { \
+ if (LIKELY(settings->on_##FOR)) { \
+ parser->state = CURRENT_STATE(); \
+ if (UNLIKELY(0 != \
+ settings->on_##FOR(parser, FOR##_mark, (LEN)))) { \
+ SET_ERRNO(HPE_CB_##FOR); \
+ } \
+ UPDATE_STATE(parser->state); \
+ \
+ /* We either errored above or got paused; get out */ \
+ if (UNLIKELY(HTTP_PARSER_ERRNO(parser) != HPE_OK)) { \
+ return (ER); \
+ } \
+ } \
+ FOR##_mark = NULL; \
+ } \
+} while (0)
+
+/* Run the data callback FOR and consume the current byte */
+#define CALLBACK_DATA(FOR) \
+ CALLBACK_DATA_(FOR, p - FOR##_mark, p - data + 1)
+
+/* Run the data callback FOR and don't consume the current byte */
+#define CALLBACK_DATA_NOADVANCE(FOR) \
+ CALLBACK_DATA_(FOR, p - FOR##_mark, p - data)
+
+/* Set the mark FOR; non-destructive if mark is already set */
+#define MARK(FOR) \
+do { \
+ if (!FOR##_mark) { \
+ FOR##_mark = p; \
+ } \
+} while (0)
+
+/* Don't allow the total size of the HTTP headers (including the status
+ * line) to exceed max_header_size. This check is here to protect
+ * embedders against denial-of-service attacks where the attacker feeds
+ * us a never-ending header that the embedder keeps buffering.
+ *
+ * This check is arguably the responsibility of embedders but we're doing
+ * it on the embedder's behalf because most won't bother and this way we
+ * make the web a little safer. max_header_size is still far bigger
+ * than any reasonable request or response so this should never affect
+ * day-to-day operation.
+ */
+#define COUNT_HEADER_SIZE(V) \
+do { \
+ nread += (uint32_t)(V); \
+ if (UNLIKELY(nread > max_header_size)) { \
+ SET_ERRNO(HPE_HEADER_OVERFLOW); \
+ goto error; \
+ } \
+} while (0)
+
+
+#define PROXY_CONNECTION "proxy-connection"
+#define CONNECTION "connection"
+#define CONTENT_LENGTH "content-length"
+#define TRANSFER_ENCODING "transfer-encoding"
+#define UPGRADE "upgrade"
+#define CHUNKED "chunked"
+#define KEEP_ALIVE "keep-alive"
+#define CLOSE "close"
+
+
+static const char *method_strings[] =
+ {
+#define XX(num, name, string) #string,
+ HTTP_METHOD_MAP(XX)
+#undef XX
+ };
+
+
+/* Tokens as defined by rfc 2616. Also lowercases them.
+ * token = 1*<any CHAR except CTLs or separators>
+ * separators = "(" | ")" | "<" | ">" | "@"
+ * | "," | ";" | ":" | "\" | <">
+ * | "/" | "[" | "]" | "?" | "="
+ * | "{" | "}" | SP | HT
+ */
+static const char tokens[256] = {
+/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
+ ' ', '!', 0, '#', '$', '%', '&', '\'',
+/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
+ 0, 0, '*', '+', 0, '-', '.', 0,
+/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
+ '0', '1', '2', '3', '4', '5', '6', '7',
+/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
+ '8', '9', 0, 0, 0, 0, 0, 0,
+/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
+ 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
+ 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
+ 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
+/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
+ 'x', 'y', 'z', 0, 0, 0, '^', '_',
+/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
+ '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
+ 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
+/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
+ 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
+/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
+ 'x', 'y', 'z', 0, '|', 0, '~', 0 };
+
+
+static const int8_t unhex[256] =
+ {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
+ ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+ };
+
+
+#if HTTP_PARSER_STRICT
+# define T(v) 0
+#else
+# define T(v) v
+#endif
+
+
+static const uint8_t normal_url_char[32] = {
+/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
+ 0 | T(2) | 0 | 0 | T(16) | 0 | 0 | 0,
+/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
+ 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
+/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
+ 0 | 2 | 4 | 0 | 16 | 32 | 64 | 128,
+/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0,
+/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
+/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
+ 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, };
+
+#undef T
+
+enum state
+ { s_dead = 1 /* important that this is > 0 */
+
+ , s_start_req_or_res
+ , s_res_or_resp_H
+ , s_start_res
+ , s_res_H
+ , s_res_HT
+ , s_res_HTT
+ , s_res_HTTP
+ , s_res_http_major
+ , s_res_http_dot
+ , s_res_http_minor
+ , s_res_http_end
+ , s_res_first_status_code
+ , s_res_status_code
+ , s_res_status_start
+ , s_res_status
+ , s_res_line_almost_done
+
+ , s_start_req
+
+ , s_req_method
+ , s_req_spaces_before_url
+ , s_req_schema
+ , s_req_schema_slash
+ , s_req_schema_slash_slash
+ , s_req_server_start
+ , s_req_server
+ , s_req_server_with_at
+ , s_req_path
+ , s_req_query_string_start
+ , s_req_query_string
+ , s_req_fragment_start
+ , s_req_fragment
+ , s_req_http_start
+ , s_req_http_H
+ , s_req_http_HT
+ , s_req_http_HTT
+ , s_req_http_HTTP
+ , s_req_http_I
+ , s_req_http_IC
+ , s_req_http_major
+ , s_req_http_dot
+ , s_req_http_minor
+ , s_req_http_end
+ , s_req_line_almost_done
+
+ , s_header_field_start
+ , s_header_field
+ , s_header_value_discard_ws
+ , s_header_value_discard_ws_almost_done
+ , s_header_value_discard_lws
+ , s_header_value_start
+ , s_header_value
+ , s_header_value_lws
+
+ , s_header_almost_done
+
+ , s_chunk_size_start
+ , s_chunk_size
+ , s_chunk_parameters
+ , s_chunk_size_almost_done
+
+ , s_headers_almost_done
+ , s_headers_done
+
+ /* Important: 's_headers_done' must be the last 'header' state. All
+ * states beyond this must be 'body' states. It is used for overflow
+ * checking. See the PARSING_HEADER() macro.
+ */
+
+ , s_chunk_data
+ , s_chunk_data_almost_done
+ , s_chunk_data_done
+
+ , s_body_identity
+ , s_body_identity_eof
+
+ , s_message_done
+ };
+
+
+#define PARSING_HEADER(state) (state <= s_headers_done)
+
+
+enum header_states
+ { h_general = 0
+ , h_C
+ , h_CO
+ , h_CON
+
+ , h_matching_connection
+ , h_matching_proxy_connection
+ , h_matching_content_length
+ , h_matching_transfer_encoding
+ , h_matching_upgrade
+
+ , h_connection
+ , h_content_length
+ , h_content_length_num
+ , h_content_length_ws
+ , h_transfer_encoding
+ , h_upgrade
+
+ , h_matching_transfer_encoding_chunked
+ , h_matching_connection_token_start
+ , h_matching_connection_keep_alive
+ , h_matching_connection_close
+ , h_matching_connection_upgrade
+ , h_matching_connection_token
+
+ , h_transfer_encoding_chunked
+ , h_connection_keep_alive
+ , h_connection_close
+ , h_connection_upgrade
+ };
+
+enum http_host_state
+ {
+ s_http_host_dead = 1
+ , s_http_userinfo_start
+ , s_http_userinfo
+ , s_http_host_start
+ , s_http_host_v6_start
+ , s_http_host
+ , s_http_host_v6
+ , s_http_host_v6_end
+ , s_http_host_v6_zone_start
+ , s_http_host_v6_zone
+ , s_http_host_port_start
+ , s_http_host_port
+};
+
+/* Macros for character classes; depends on strict-mode */
+#define CR '\r'
+#define LF '\n'
+#define LOWER(c) (unsigned char)(c | 0x20)
+#define IS_ALPHA(c) (LOWER(c) >= 'a' && LOWER(c) <= 'z')
+#define IS_NUM(c) ((c) >= '0' && (c) <= '9')
+#define IS_ALPHANUM(c) (IS_ALPHA(c) || IS_NUM(c))
+#define IS_HEX(c) (IS_NUM(c) || (LOWER(c) >= 'a' && LOWER(c) <= 'f'))
+#define IS_MARK(c) ((c) == '-' || (c) == '_' || (c) == '.' || \
+ (c) == '!' || (c) == '~' || (c) == '*' || (c) == '\'' || (c) == '(' || \
+ (c) == ')')
+#define IS_USERINFO_CHAR(c) (IS_ALPHANUM(c) || IS_MARK(c) || (c) == '%' || \
+ (c) == ';' || (c) == ':' || (c) == '&' || (c) == '=' || (c) == '+' || \
+ (c) == '$' || (c) == ',')
+
+#define STRICT_TOKEN(c) ((c == ' ') ? 0 : tokens[(unsigned char)c])
+
+#if HTTP_PARSER_STRICT
+#define TOKEN(c) STRICT_TOKEN(c)
+#define IS_URL_CHAR(c) (BIT_AT(normal_url_char, (unsigned char)c))
+#define IS_HOST_CHAR(c) (IS_ALPHANUM(c) || (c) == '.' || (c) == '-')
+#else
+#define TOKEN(c) tokens[(unsigned char)c]
+#define IS_URL_CHAR(c) \
+ (BIT_AT(normal_url_char, (unsigned char)c) || ((c) & 0x80))
+#define IS_HOST_CHAR(c) \
+ (IS_ALPHANUM(c) || (c) == '.' || (c) == '-' || (c) == '_')
+#endif
+
+/**
+ * Verify that a char is a valid visible (printable) US-ASCII
+ * character or %x80-FF
+ **/
+#define IS_HEADER_CHAR(ch) \
+ (ch == CR || ch == LF || ch == 9 || ((unsigned char)ch > 31 && ch != 127))
+
+#define start_state (parser->type == HTTP_REQUEST ? s_start_req : s_start_res)
+
+
+#if HTTP_PARSER_STRICT
+# define STRICT_CHECK(cond) \
+do { \
+ if (cond) { \
+ SET_ERRNO(HPE_STRICT); \
+ goto error; \
+ } \
+} while (0)
+# define NEW_MESSAGE() (http_should_keep_alive(parser) ? start_state : s_dead)
+#else
+# define STRICT_CHECK(cond)
+# define NEW_MESSAGE() start_state
+#endif
+
+
+/* Map errno values to strings for human-readable output */
+#define HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s },
+static struct {
+ const char *name;
+ const char *description;
+} http_strerror_tab[] = {
+ HTTP_ERRNO_MAP(HTTP_STRERROR_GEN)
+};
+#undef HTTP_STRERROR_GEN
+
+int http_message_needs_eof(const http_parser *parser);
+
+/* Our URL parser.
+ *
+ * This is designed to be shared by http_parser_execute() for URL validation,
+ * hence it has a state transition + byte-for-byte interface. In addition, it
+ * is meant to be embedded in http_parser_parse_url(), which does the dirty
+ * work of turning state transitions URL components for its API.
+ *
+ * This function should only be invoked with non-space characters. It is
+ * assumed that the caller cares about (and can detect) the transition between
+ * URL and non-URL states by looking for these.
+ */
+static enum state
+parse_url_char(enum state s, const char ch)
+{
+ if (ch == ' ' || ch == '\r' || ch == '\n') {
+ return s_dead;
+ }
+
+#if HTTP_PARSER_STRICT
+ if (ch == '\t' || ch == '\f') {
+ return s_dead;
+ }
+#endif
+
+ switch (s) {
+ case s_req_spaces_before_url:
+ /* Proxied requests are followed by scheme of an absolute URI (alpha).
+ * All methods except CONNECT are followed by '/' or '*'.
+ */
+
+ if (ch == '/' || ch == '*') {
+ return s_req_path;
+ }
+
+ if (IS_ALPHA(ch)) {
+ return s_req_schema;
+ }
+
+ break;
+
+ case s_req_schema:
+ if (IS_ALPHA(ch)) {
+ return s;
+ }
+
+ if (ch == ':') {
+ return s_req_schema_slash;
+ }
+
+ break;
+
+ case s_req_schema_slash:
+ if (ch == '/') {
+ return s_req_schema_slash_slash;
+ }
+
+ break;
+
+ case s_req_schema_slash_slash:
+ if (ch == '/') {
+ return s_req_server_start;
+ }
+
+ break;
+
+ case s_req_server_with_at:
+ if (ch == '@') {
+ return s_dead;
+ }
+
+ /* fall through */
+ case s_req_server_start:
+ case s_req_server:
+ if (ch == '/') {
+ return s_req_path;
+ }
+
+ if (ch == '?') {
+ return s_req_query_string_start;
+ }
+
+ if (ch == '@') {
+ return s_req_server_with_at;
+ }
+
+ if (IS_USERINFO_CHAR(ch) || ch == '[' || ch == ']') {
+ return s_req_server;
+ }
+
+ break;
+
+ case s_req_path:
+ if (IS_URL_CHAR(ch)) {
+ return s;
+ }
+
+ switch (ch) {
+ case '?':
+ return s_req_query_string_start;
+
+ case '#':
+ return s_req_fragment_start;
+ }
+
+ break;
+
+ case s_req_query_string_start:
+ case s_req_query_string:
+ if (IS_URL_CHAR(ch)) {
+ return s_req_query_string;
+ }
+
+ switch (ch) {
+ case '?':
+ /* allow extra '?' in query string */
+ return s_req_query_string;
+
+ case '#':
+ return s_req_fragment_start;
+ }
+
+ break;
+
+ case s_req_fragment_start:
+ if (IS_URL_CHAR(ch)) {
+ return s_req_fragment;
+ }
+
+ switch (ch) {
+ case '?':
+ return s_req_fragment;
+
+ case '#':
+ return s;
+ }
+
+ break;
+
+ case s_req_fragment:
+ if (IS_URL_CHAR(ch)) {
+ return s;
+ }
+
+ switch (ch) {
+ case '?':
+ case '#':
+ return s;
+ }
+
+ break;
+
+ default:
+ break;
+ }
+
+ /* We should never fall out of the switch above unless there's an error */
+ return s_dead;
+}
+
+size_t http_parser_execute (http_parser *parser,
+ const http_parser_settings *settings,
+ const char *data,
+ size_t len)
+{
+ char c, ch;
+ int8_t unhex_val;
+ const char *p = data;
+ const char *header_field_mark = 0;
+ const char *header_value_mark = 0;
+ const char *url_mark = 0;
+ const char *body_mark = 0;
+ const char *status_mark = 0;
+ enum state p_state = (enum state) parser->state;
+ const unsigned int lenient = parser->lenient_http_headers;
+ uint32_t nread = parser->nread;
+
+ /* We're in an error state. Don't bother doing anything. */
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) {
+ return 0;
+ }
+
+ if (len == 0) {
+ switch (CURRENT_STATE()) {
+ case s_body_identity_eof:
+ /* Use of CALLBACK_NOTIFY() here would erroneously return 1 byte read if
+ * we got paused.
+ */
+ CALLBACK_NOTIFY_NOADVANCE(message_complete);
+ return 0;
+
+ case s_dead:
+ case s_start_req_or_res:
+ case s_start_res:
+ case s_start_req:
+ return 0;
+
+ default:
+ SET_ERRNO(HPE_INVALID_EOF_STATE);
+ return 1;
+ }
+ }
+
+
+ if (CURRENT_STATE() == s_header_field)
+ header_field_mark = data;
+ if (CURRENT_STATE() == s_header_value)
+ header_value_mark = data;
+ switch (CURRENT_STATE()) {
+ case s_req_path:
+ case s_req_schema:
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ case s_req_server:
+ case s_req_server_with_at:
+ case s_req_query_string_start:
+ case s_req_query_string:
+ case s_req_fragment_start:
+ case s_req_fragment:
+ url_mark = data;
+ break;
+ case s_res_status:
+ status_mark = data;
+ break;
+ default:
+ break;
+ }
+
+ for (p=data; p != data + len; p++) {
+ ch = *p;
+
+ if (PARSING_HEADER(CURRENT_STATE()))
+ COUNT_HEADER_SIZE(1);
+
+reexecute:
+ switch (CURRENT_STATE()) {
+
+ case s_dead:
+ /* this state is used after a 'Connection: close' message
+ * the parser will error out if it reads another message
+ */
+ if (LIKELY(ch == CR || ch == LF))
+ break;
+
+ SET_ERRNO(HPE_CLOSED_CONNECTION);
+ goto error;
+
+ case s_start_req_or_res:
+ {
+ if (ch == CR || ch == LF)
+ break;
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ if (ch == 'H') {
+ UPDATE_STATE(s_res_or_resp_H);
+
+ CALLBACK_NOTIFY(message_begin);
+ } else {
+ parser->type = HTTP_REQUEST;
+ UPDATE_STATE(s_start_req);
+ REEXECUTE();
+ }
+
+ break;
+ }
+
+ case s_res_or_resp_H:
+ if (ch == 'T') {
+ parser->type = HTTP_RESPONSE;
+ UPDATE_STATE(s_res_HT);
+ } else {
+ if (UNLIKELY(ch != 'E')) {
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+
+ parser->type = HTTP_REQUEST;
+ parser->method = HTTP_HEAD;
+ parser->index = 2;
+ UPDATE_STATE(s_req_method);
+ }
+ break;
+
+ case s_start_res:
+ {
+ if (ch == CR || ch == LF)
+ break;
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ if (ch == 'H') {
+ UPDATE_STATE(s_res_H);
+ } else {
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+
+ CALLBACK_NOTIFY(message_begin);
+ break;
+ }
+
+ case s_res_H:
+ STRICT_CHECK(ch != 'T');
+ UPDATE_STATE(s_res_HT);
+ break;
+
+ case s_res_HT:
+ STRICT_CHECK(ch != 'T');
+ UPDATE_STATE(s_res_HTT);
+ break;
+
+ case s_res_HTT:
+ STRICT_CHECK(ch != 'P');
+ UPDATE_STATE(s_res_HTTP);
+ break;
+
+ case s_res_HTTP:
+ STRICT_CHECK(ch != '/');
+ UPDATE_STATE(s_res_http_major);
+ break;
+
+ case s_res_http_major:
+ if (UNLIKELY(!IS_NUM(ch))) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major = ch - '0';
+ UPDATE_STATE(s_res_http_dot);
+ break;
+
+ case s_res_http_dot:
+ {
+ if (UNLIKELY(ch != '.')) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ UPDATE_STATE(s_res_http_minor);
+ break;
+ }
+
+ case s_res_http_minor:
+ if (UNLIKELY(!IS_NUM(ch))) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor = ch - '0';
+ UPDATE_STATE(s_res_http_end);
+ break;
+
+ case s_res_http_end:
+ {
+ if (UNLIKELY(ch != ' ')) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ UPDATE_STATE(s_res_first_status_code);
+ break;
+ }
+
+ case s_res_first_status_code:
+ {
+ if (!IS_NUM(ch)) {
+ if (ch == ' ') {
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+ parser->status_code = ch - '0';
+ UPDATE_STATE(s_res_status_code);
+ break;
+ }
+
+ case s_res_status_code:
+ {
+ if (!IS_NUM(ch)) {
+ switch (ch) {
+ case ' ':
+ UPDATE_STATE(s_res_status_start);
+ break;
+ case CR:
+ case LF:
+ UPDATE_STATE(s_res_status_start);
+ REEXECUTE();
+ break;
+ default:
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+ break;
+ }
+
+ parser->status_code *= 10;
+ parser->status_code += ch - '0';
+
+ if (UNLIKELY(parser->status_code > 999)) {
+ SET_ERRNO(HPE_INVALID_STATUS);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_res_status_start:
+ {
+ MARK(status);
+ UPDATE_STATE(s_res_status);
+ parser->index = 0;
+
+ if (ch == CR || ch == LF)
+ REEXECUTE();
+
+ break;
+ }
+
+ case s_res_status:
+ if (ch == CR) {
+ UPDATE_STATE(s_res_line_almost_done);
+ CALLBACK_DATA(status);
+ break;
+ }
+
+ if (ch == LF) {
+ UPDATE_STATE(s_header_field_start);
+ CALLBACK_DATA(status);
+ break;
+ }
+
+ break;
+
+ case s_res_line_almost_done:
+ STRICT_CHECK(ch != LF);
+ UPDATE_STATE(s_header_field_start);
+ break;
+
+ case s_start_req:
+ {
+ if (ch == CR || ch == LF)
+ break;
+ parser->flags = 0;
+ parser->content_length = ULLONG_MAX;
+
+ if (UNLIKELY(!IS_ALPHA(ch))) {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ parser->method = (enum http_method) 0;
+ parser->index = 1;
+ switch (ch) {
+ case 'A': parser->method = HTTP_ACL; break;
+ case 'B': parser->method = HTTP_BIND; break;
+ case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break;
+ case 'D': parser->method = HTTP_DELETE; break;
+ case 'G': parser->method = HTTP_GET; break;
+ case 'H': parser->method = HTTP_HEAD; break;
+ case 'L': parser->method = HTTP_LOCK; /* or LINK */ break;
+ case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH, MKCALENDAR */ break;
+ case 'N': parser->method = HTTP_NOTIFY; break;
+ case 'O': parser->method = HTTP_OPTIONS; break;
+ case 'P': parser->method = HTTP_POST;
+ /* or PROPFIND|PROPPATCH|PUT|PATCH|PURGE */
+ break;
+ case 'R': parser->method = HTTP_REPORT; /* or REBIND */ break;
+ case 'S': parser->method = HTTP_SUBSCRIBE; /* or SEARCH, SOURCE */ break;
+ case 'T': parser->method = HTTP_TRACE; break;
+ case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE, UNBIND, UNLINK */ break;
+ default:
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ UPDATE_STATE(s_req_method);
+
+ CALLBACK_NOTIFY(message_begin);
+
+ break;
+ }
+
+ case s_req_method:
+ {
+ const char *matcher;
+ if (UNLIKELY(ch == '\0')) {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ matcher = method_strings[parser->method];
+ if (ch == ' ' && matcher[parser->index] == '\0') {
+ UPDATE_STATE(s_req_spaces_before_url);
+ } else if (ch == matcher[parser->index]) {
+ ; /* nada */
+ } else if ((ch >= 'A' && ch <= 'Z') || ch == '-') {
+
+ switch (parser->method << 16 | parser->index << 8 | ch) {
+#define XX(meth, pos, ch, new_meth) \
+ case (HTTP_##meth << 16 | pos << 8 | ch): \
+ parser->method = HTTP_##new_meth; break;
+
+ XX(POST, 1, 'U', PUT)
+ XX(POST, 1, 'A', PATCH)
+ XX(POST, 1, 'R', PROPFIND)
+ XX(PUT, 2, 'R', PURGE)
+ XX(CONNECT, 1, 'H', CHECKOUT)
+ XX(CONNECT, 2, 'P', COPY)
+ XX(MKCOL, 1, 'O', MOVE)
+ XX(MKCOL, 1, 'E', MERGE)
+ XX(MKCOL, 1, '-', MSEARCH)
+ XX(MKCOL, 2, 'A', MKACTIVITY)
+ XX(MKCOL, 3, 'A', MKCALENDAR)
+ XX(SUBSCRIBE, 1, 'E', SEARCH)
+ XX(SUBSCRIBE, 1, 'O', SOURCE)
+ XX(REPORT, 2, 'B', REBIND)
+ XX(PROPFIND, 4, 'P', PROPPATCH)
+ XX(LOCK, 1, 'I', LINK)
+ XX(UNLOCK, 2, 'S', UNSUBSCRIBE)
+ XX(UNLOCK, 2, 'B', UNBIND)
+ XX(UNLOCK, 3, 'I', UNLINK)
+#undef XX
+ default:
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+ } else {
+ SET_ERRNO(HPE_INVALID_METHOD);
+ goto error;
+ }
+
+ ++parser->index;
+ break;
+ }
+
+ case s_req_spaces_before_url:
+ {
+ if (ch == ' ') break;
+
+ MARK(url);
+ if (parser->method == HTTP_CONNECT) {
+ UPDATE_STATE(s_req_server_start);
+ }
+
+ UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch));
+ if (UNLIKELY(CURRENT_STATE() == s_dead)) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+
+ break;
+ }
+
+ case s_req_schema:
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ {
+ switch (ch) {
+ /* No whitespace allowed here */
+ case ' ':
+ case CR:
+ case LF:
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ default:
+ UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch));
+ if (UNLIKELY(CURRENT_STATE() == s_dead)) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+ }
+
+ break;
+ }
+
+ case s_req_server:
+ case s_req_server_with_at:
+ case s_req_path:
+ case s_req_query_string_start:
+ case s_req_query_string:
+ case s_req_fragment_start:
+ case s_req_fragment:
+ {
+ switch (ch) {
+ case ' ':
+ UPDATE_STATE(s_req_http_start);
+ CALLBACK_DATA(url);
+ break;
+ case CR:
+ case LF:
+ parser->http_major = 0;
+ parser->http_minor = 9;
+ UPDATE_STATE((ch == CR) ?
+ s_req_line_almost_done :
+ s_header_field_start);
+ CALLBACK_DATA(url);
+ break;
+ default:
+ UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch));
+ if (UNLIKELY(CURRENT_STATE() == s_dead)) {
+ SET_ERRNO(HPE_INVALID_URL);
+ goto error;
+ }
+ }
+ break;
+ }
+
+ case s_req_http_start:
+ switch (ch) {
+ case ' ':
+ break;
+ case 'H':
+ UPDATE_STATE(s_req_http_H);
+ break;
+ case 'I':
+ if (parser->method == HTTP_SOURCE) {
+ UPDATE_STATE(s_req_http_I);
+ break;
+ }
+ /* fall through */
+ default:
+ SET_ERRNO(HPE_INVALID_CONSTANT);
+ goto error;
+ }
+ break;
+
+ case s_req_http_H:
+ STRICT_CHECK(ch != 'T');
+ UPDATE_STATE(s_req_http_HT);
+ break;
+
+ case s_req_http_HT:
+ STRICT_CHECK(ch != 'T');
+ UPDATE_STATE(s_req_http_HTT);
+ break;
+
+ case s_req_http_HTT:
+ STRICT_CHECK(ch != 'P');
+ UPDATE_STATE(s_req_http_HTTP);
+ break;
+
+ case s_req_http_I:
+ STRICT_CHECK(ch != 'C');
+ UPDATE_STATE(s_req_http_IC);
+ break;
+
+ case s_req_http_IC:
+ STRICT_CHECK(ch != 'E');
+ UPDATE_STATE(s_req_http_HTTP); /* Treat "ICE" as "HTTP". */
+ break;
+
+ case s_req_http_HTTP:
+ STRICT_CHECK(ch != '/');
+ UPDATE_STATE(s_req_http_major);
+ break;
+
+ case s_req_http_major:
+ if (UNLIKELY(!IS_NUM(ch))) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_major = ch - '0';
+ UPDATE_STATE(s_req_http_dot);
+ break;
+
+ case s_req_http_dot:
+ {
+ if (UNLIKELY(ch != '.')) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ UPDATE_STATE(s_req_http_minor);
+ break;
+ }
+
+ case s_req_http_minor:
+ if (UNLIKELY(!IS_NUM(ch))) {
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ }
+
+ parser->http_minor = ch - '0';
+ UPDATE_STATE(s_req_http_end);
+ break;
+
+ case s_req_http_end:
+ {
+ if (ch == CR) {
+ UPDATE_STATE(s_req_line_almost_done);
+ break;
+ }
+
+ if (ch == LF) {
+ UPDATE_STATE(s_header_field_start);
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_VERSION);
+ goto error;
+ break;
+ }
+
+ /* end of request line */
+ case s_req_line_almost_done:
+ {
+ if (UNLIKELY(ch != LF)) {
+ SET_ERRNO(HPE_LF_EXPECTED);
+ goto error;
+ }
+
+ UPDATE_STATE(s_header_field_start);
+ break;
+ }
+
+ case s_header_field_start:
+ {
+ if (ch == CR) {
+ UPDATE_STATE(s_headers_almost_done);
+ break;
+ }
+
+ if (ch == LF) {
+ /* they might be just sending \n instead of \r\n so this would be
+ * the second \n to denote the end of headers*/
+ UPDATE_STATE(s_headers_almost_done);
+ REEXECUTE();
+ }
+
+ c = TOKEN(ch);
+
+ if (UNLIKELY(!c)) {
+ SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
+ goto error;
+ }
+
+ MARK(header_field);
+
+ parser->index = 0;
+ UPDATE_STATE(s_header_field);
+
+ switch (c) {
+ case 'c':
+ parser->header_state = h_C;
+ break;
+
+ case 'p':
+ parser->header_state = h_matching_proxy_connection;
+ break;
+
+ case 't':
+ parser->header_state = h_matching_transfer_encoding;
+ break;
+
+ case 'u':
+ parser->header_state = h_matching_upgrade;
+ break;
+
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+ }
+
+ case s_header_field:
+ {
+ const char* start = p;
+ for (; p != data + len; p++) {
+ ch = *p;
+ c = TOKEN(ch);
+
+ if (!c)
+ break;
+
+ switch (parser->header_state) {
+ case h_general: {
+ size_t left = data + len - p;
+ const char* pe = p + MIN(left, max_header_size);
+ while (p+1 < pe && TOKEN(p[1])) {
+ p++;
+ }
+ break;
+ }
+
+ case h_C:
+ parser->index++;
+ parser->header_state = (c == 'o' ? h_CO : h_general);
+ break;
+
+ case h_CO:
+ parser->index++;
+ parser->header_state = (c == 'n' ? h_CON : h_general);
+ break;
+
+ case h_CON:
+ parser->index++;
+ switch (c) {
+ case 'n':
+ parser->header_state = h_matching_connection;
+ break;
+ case 't':
+ parser->header_state = h_matching_content_length;
+ break;
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+
+ /* connection */
+
+ case h_matching_connection:
+ parser->index++;
+ if (parser->index > sizeof(CONNECTION)-1
+ || c != CONNECTION[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CONNECTION)-2) {
+ parser->header_state = h_connection;
+ }
+ break;
+
+ /* proxy-connection */
+
+ case h_matching_proxy_connection:
+ parser->index++;
+ if (parser->index > sizeof(PROXY_CONNECTION)-1
+ || c != PROXY_CONNECTION[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(PROXY_CONNECTION)-2) {
+ parser->header_state = h_connection;
+ }
+ break;
+
+ /* content-length */
+
+ case h_matching_content_length:
+ parser->index++;
+ if (parser->index > sizeof(CONTENT_LENGTH)-1
+ || c != CONTENT_LENGTH[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(CONTENT_LENGTH)-2) {
+ parser->header_state = h_content_length;
+ }
+ break;
+
+ /* transfer-encoding */
+
+ case h_matching_transfer_encoding:
+ parser->index++;
+ if (parser->index > sizeof(TRANSFER_ENCODING)-1
+ || c != TRANSFER_ENCODING[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) {
+ parser->header_state = h_transfer_encoding;
+ }
+ break;
+
+ /* upgrade */
+
+ case h_matching_upgrade:
+ parser->index++;
+ if (parser->index > sizeof(UPGRADE)-1
+ || c != UPGRADE[parser->index]) {
+ parser->header_state = h_general;
+ } else if (parser->index == sizeof(UPGRADE)-2) {
+ parser->header_state = h_upgrade;
+ }
+ break;
+
+ case h_connection:
+ case h_content_length:
+ case h_transfer_encoding:
+ case h_upgrade:
+ if (ch != ' ') parser->header_state = h_general;
+ break;
+
+ default:
+ assert(0 && "Unknown header_state");
+ break;
+ }
+ }
+
+ if (p == data + len) {
+ --p;
+ COUNT_HEADER_SIZE(p - start);
+ break;
+ }
+
+ COUNT_HEADER_SIZE(p - start);
+
+ if (ch == ':') {
+ UPDATE_STATE(s_header_value_discard_ws);
+ CALLBACK_DATA(header_field);
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
+ goto error;
+ }
+
+ case s_header_value_discard_ws:
+ if (ch == ' ' || ch == '\t') break;
+
+ if (ch == CR) {
+ UPDATE_STATE(s_header_value_discard_ws_almost_done);
+ break;
+ }
+
+ if (ch == LF) {
+ UPDATE_STATE(s_header_value_discard_lws);
+ break;
+ }
+
+ /* fall through */
+
+ case s_header_value_start:
+ {
+ MARK(header_value);
+
+ UPDATE_STATE(s_header_value);
+ parser->index = 0;
+
+ c = LOWER(ch);
+
+ switch (parser->header_state) {
+ case h_upgrade:
+ parser->flags |= F_UPGRADE;
+ parser->header_state = h_general;
+ break;
+
+ case h_transfer_encoding:
+ /* looking for 'Transfer-Encoding: chunked' */
+ if ('c' == c) {
+ parser->header_state = h_matching_transfer_encoding_chunked;
+ } else {
+ parser->header_state = h_general;
+ }
+ break;
+
+ case h_content_length:
+ if (UNLIKELY(!IS_NUM(ch))) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ if (parser->flags & F_CONTENTLENGTH) {
+ SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH);
+ goto error;
+ }
+
+ parser->flags |= F_CONTENTLENGTH;
+ parser->content_length = ch - '0';
+ parser->header_state = h_content_length_num;
+ break;
+
+ /* when obsolete line folding is encountered for content length
+ * continue to the s_header_value state */
+ case h_content_length_ws:
+ break;
+
+ case h_connection:
+ /* looking for 'Connection: keep-alive' */
+ if (c == 'k') {
+ parser->header_state = h_matching_connection_keep_alive;
+ /* looking for 'Connection: close' */
+ } else if (c == 'c') {
+ parser->header_state = h_matching_connection_close;
+ } else if (c == 'u') {
+ parser->header_state = h_matching_connection_upgrade;
+ } else {
+ parser->header_state = h_matching_connection_token;
+ }
+ break;
+
+ /* Multi-value `Connection` header */
+ case h_matching_connection_token_start:
+ break;
+
+ default:
+ parser->header_state = h_general;
+ break;
+ }
+ break;
+ }
+
+ case s_header_value:
+ {
+ const char* start = p;
+ enum header_states h_state = (enum header_states) parser->header_state;
+ for (; p != data + len; p++) {
+ ch = *p;
+ if (ch == CR) {
+ UPDATE_STATE(s_header_almost_done);
+ parser->header_state = h_state;
+ CALLBACK_DATA(header_value);
+ break;
+ }
+
+ if (ch == LF) {
+ UPDATE_STATE(s_header_almost_done);
+ COUNT_HEADER_SIZE(p - start);
+ parser->header_state = h_state;
+ CALLBACK_DATA_NOADVANCE(header_value);
+ REEXECUTE();
+ }
+
+ if (!lenient && !IS_HEADER_CHAR(ch)) {
+ SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
+ goto error;
+ }
+
+ c = LOWER(ch);
+
+ switch (h_state) {
+ case h_general:
+ {
+ size_t left = data + len - p;
+ const char* pe = p + MIN(left, max_header_size);
+
+ for (; p != pe; p++) {
+ ch = *p;
+ if (ch == CR || ch == LF) {
+ --p;
+ break;
+ }
+ if (!lenient && !IS_HEADER_CHAR(ch)) {
+ SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
+ goto error;
+ }
+ }
+ if (p == data + len)
+ --p;
+ break;
+ }
+
+ case h_connection:
+ case h_transfer_encoding:
+ assert(0 && "Shouldn't get here.");
+ break;
+
+ case h_content_length:
+ if (ch == ' ') break;
+ h_state = h_content_length_num;
+ /* fall through */
+
+ case h_content_length_num:
+ {
+ uint64_t t;
+
+ if (ch == ' ') {
+ h_state = h_content_length_ws;
+ break;
+ }
+
+ if (UNLIKELY(!IS_NUM(ch))) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ parser->header_state = h_state;
+ goto error;
+ }
+
+ t = parser->content_length;
+ t *= 10;
+ t += ch - '0';
+
+ /* Overflow? Test against a conservative limit for simplicity. */
+ if (UNLIKELY((ULLONG_MAX - 10) / 10 < parser->content_length)) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ parser->header_state = h_state;
+ goto error;
+ }
+
+ parser->content_length = t;
+ break;
+ }
+
+ case h_content_length_ws:
+ if (ch == ' ') break;
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ parser->header_state = h_state;
+ goto error;
+
+ /* Transfer-Encoding: chunked */
+ case h_matching_transfer_encoding_chunked:
+ parser->index++;
+ if (parser->index > sizeof(CHUNKED)-1
+ || c != CHUNKED[parser->index]) {
+ h_state = h_general;
+ } else if (parser->index == sizeof(CHUNKED)-2) {
+ h_state = h_transfer_encoding_chunked;
+ }
+ break;
+
+ case h_matching_connection_token_start:
+ /* looking for 'Connection: keep-alive' */
+ if (c == 'k') {
+ h_state = h_matching_connection_keep_alive;
+ /* looking for 'Connection: close' */
+ } else if (c == 'c') {
+ h_state = h_matching_connection_close;
+ } else if (c == 'u') {
+ h_state = h_matching_connection_upgrade;
+ } else if (STRICT_TOKEN(c)) {
+ h_state = h_matching_connection_token;
+ } else if (c == ' ' || c == '\t') {
+ /* Skip lws */
+ } else {
+ h_state = h_general;
+ }
+ break;
+
+ /* looking for 'Connection: keep-alive' */
+ case h_matching_connection_keep_alive:
+ parser->index++;
+ if (parser->index > sizeof(KEEP_ALIVE)-1
+ || c != KEEP_ALIVE[parser->index]) {
+ h_state = h_matching_connection_token;
+ } else if (parser->index == sizeof(KEEP_ALIVE)-2) {
+ h_state = h_connection_keep_alive;
+ }
+ break;
+
+ /* looking for 'Connection: close' */
+ case h_matching_connection_close:
+ parser->index++;
+ if (parser->index > sizeof(CLOSE)-1 || c != CLOSE[parser->index]) {
+ h_state = h_matching_connection_token;
+ } else if (parser->index == sizeof(CLOSE)-2) {
+ h_state = h_connection_close;
+ }
+ break;
+
+ /* looking for 'Connection: upgrade' */
+ case h_matching_connection_upgrade:
+ parser->index++;
+ if (parser->index > sizeof(UPGRADE) - 1 ||
+ c != UPGRADE[parser->index]) {
+ h_state = h_matching_connection_token;
+ } else if (parser->index == sizeof(UPGRADE)-2) {
+ h_state = h_connection_upgrade;
+ }
+ break;
+
+ case h_matching_connection_token:
+ if (ch == ',') {
+ h_state = h_matching_connection_token_start;
+ parser->index = 0;
+ }
+ break;
+
+ case h_transfer_encoding_chunked:
+ if (ch != ' ') h_state = h_general;
+ break;
+
+ case h_connection_keep_alive:
+ case h_connection_close:
+ case h_connection_upgrade:
+ if (ch == ',') {
+ if (h_state == h_connection_keep_alive) {
+ parser->flags |= F_CONNECTION_KEEP_ALIVE;
+ } else if (h_state == h_connection_close) {
+ parser->flags |= F_CONNECTION_CLOSE;
+ } else if (h_state == h_connection_upgrade) {
+ parser->flags |= F_CONNECTION_UPGRADE;
+ }
+ h_state = h_matching_connection_token_start;
+ parser->index = 0;
+ } else if (ch != ' ') {
+ h_state = h_matching_connection_token;
+ }
+ break;
+
+ default:
+ UPDATE_STATE(s_header_value);
+ h_state = h_general;
+ break;
+ }
+ }
+ parser->header_state = h_state;
+
+ if (p == data + len)
+ --p;
+
+ COUNT_HEADER_SIZE(p - start);
+ break;
+ }
+
+ case s_header_almost_done:
+ {
+ if (UNLIKELY(ch != LF)) {
+ SET_ERRNO(HPE_LF_EXPECTED);
+ goto error;
+ }
+
+ UPDATE_STATE(s_header_value_lws);
+ break;
+ }
+
+ case s_header_value_lws:
+ {
+ if (ch == ' ' || ch == '\t') {
+ if (parser->header_state == h_content_length_num) {
+ /* treat obsolete line folding as space */
+ parser->header_state = h_content_length_ws;
+ }
+ UPDATE_STATE(s_header_value_start);
+ REEXECUTE();
+ }
+
+ /* finished the header */
+ switch (parser->header_state) {
+ case h_connection_keep_alive:
+ parser->flags |= F_CONNECTION_KEEP_ALIVE;
+ break;
+ case h_connection_close:
+ parser->flags |= F_CONNECTION_CLOSE;
+ break;
+ case h_transfer_encoding_chunked:
+ parser->flags |= F_CHUNKED;
+ break;
+ case h_connection_upgrade:
+ parser->flags |= F_CONNECTION_UPGRADE;
+ break;
+ default:
+ break;
+ }
+
+ UPDATE_STATE(s_header_field_start);
+ REEXECUTE();
+ }
+ /* fall through */
+ case s_header_value_discard_ws_almost_done:
+ {
+ STRICT_CHECK(ch != LF);
+ UPDATE_STATE(s_header_value_discard_lws);
+ break;
+ }
+
+ case s_header_value_discard_lws:
+ {
+ if (ch == ' ' || ch == '\t') {
+ UPDATE_STATE(s_header_value_discard_ws);
+ break;
+ } else {
+ switch (parser->header_state) {
+ case h_connection_keep_alive:
+ parser->flags |= F_CONNECTION_KEEP_ALIVE;
+ break;
+ case h_connection_close:
+ parser->flags |= F_CONNECTION_CLOSE;
+ break;
+ case h_connection_upgrade:
+ parser->flags |= F_CONNECTION_UPGRADE;
+ break;
+ case h_transfer_encoding_chunked:
+ parser->flags |= F_CHUNKED;
+ break;
+ case h_content_length:
+ /* do not allow empty content length */
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ break;
+ default:
+ break;
+ }
+
+ /* header value was empty */
+ MARK(header_value);
+ UPDATE_STATE(s_header_field_start);
+ CALLBACK_DATA_NOADVANCE(header_value);
+ REEXECUTE();
+ }
+ }
+ /* fall through */
+ case s_headers_almost_done:
+ {
+ STRICT_CHECK(ch != LF);
+
+ if (parser->flags & F_TRAILING) {
+ /* End of a chunked request */
+ UPDATE_STATE(s_message_done);
+ CALLBACK_NOTIFY_NOADVANCE(chunk_complete);
+ REEXECUTE();
+ }
+
+ /* Cannot use chunked encoding and a content-length header together
+ per the HTTP specification. */
+ if ((parser->flags & F_CHUNKED) &&
+ (parser->flags & F_CONTENTLENGTH)) {
+ SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH);
+ goto error;
+ }
+
+ UPDATE_STATE(s_headers_done);
+
+ /* Set this here so that on_headers_complete() callbacks can see it */
+ if ((parser->flags & F_UPGRADE) &&
+ (parser->flags & F_CONNECTION_UPGRADE)) {
+ /* For responses, "Upgrade: foo" and "Connection: upgrade" are
+ * mandatory only when it is a 101 Switching Protocols response,
+ * otherwise it is purely informational, to announce support.
+ */
+ parser->upgrade =
+ (parser->type == HTTP_REQUEST || parser->status_code == 101);
+ } else {
+ parser->upgrade = (parser->method == HTTP_CONNECT);
+ }
+
+ /* Here we call the headers_complete callback. This is somewhat
+ * different than other callbacks because if the user returns 1, we
+ * will interpret that as saying that this message has no body. This
+ * is needed for the annoying case of recieving a response to a HEAD
+ * request.
+ *
+ * We'd like to use CALLBACK_NOTIFY_NOADVANCE() here but we cannot, so
+ * we have to simulate it by handling a change in errno below.
+ */
+ if (settings->on_headers_complete) {
+ switch (settings->on_headers_complete(parser)) {
+ case 0:
+ break;
+
+ case 2:
+ parser->upgrade = 1;
+
+ /* fall through */
+ case 1:
+ parser->flags |= F_SKIPBODY;
+ break;
+
+ default:
+ SET_ERRNO(HPE_CB_headers_complete);
+ RETURN(p - data); /* Error */
+ }
+ }
+
+ if (HTTP_PARSER_ERRNO(parser) != HPE_OK) {
+ RETURN(p - data);
+ }
+
+ REEXECUTE();
+ }
+ /* fall through */
+ case s_headers_done:
+ {
+ int hasBody;
+ STRICT_CHECK(ch != LF);
+
+ parser->nread = 0;
+ nread = 0;
+
+ hasBody = (parser->flags & F_CHUNKED) ||
+ (parser->content_length > 0 && parser->content_length != ULLONG_MAX);
+ if (parser->upgrade && (parser->method == HTTP_CONNECT ||
+ (parser->flags & F_SKIPBODY) || !hasBody)) {
+ /* Exit, the rest of the message is in a different protocol. */
+ UPDATE_STATE(NEW_MESSAGE());
+ CALLBACK_NOTIFY(message_complete);
+ RETURN((p - data) + 1);
+ }
+
+ if (parser->flags & F_SKIPBODY) {
+ UPDATE_STATE(NEW_MESSAGE());
+ CALLBACK_NOTIFY(message_complete);
+ } else if (parser->flags & F_CHUNKED) {
+ /* chunked encoding - ignore Content-Length header */
+ UPDATE_STATE(s_chunk_size_start);
+ } else {
+ if (parser->content_length == 0) {
+ /* Content-Length header given but zero: Content-Length: 0\r\n */
+ UPDATE_STATE(NEW_MESSAGE());
+ CALLBACK_NOTIFY(message_complete);
+ } else if (parser->content_length != ULLONG_MAX) {
+ /* Content-Length header given and non-zero */
+ UPDATE_STATE(s_body_identity);
+ } else {
+ if (!http_message_needs_eof(parser)) {
+ /* Assume content-length 0 - read the next */
+ UPDATE_STATE(NEW_MESSAGE());
+ CALLBACK_NOTIFY(message_complete);
+ } else {
+ /* Read body until EOF */
+ UPDATE_STATE(s_body_identity_eof);
+ }
+ }
+ }
+
+ break;
+ }
+
+ case s_body_identity:
+ {
+ uint64_t to_read = MIN(parser->content_length,
+ (uint64_t) ((data + len) - p));
+
+ assert(parser->content_length != 0
+ && parser->content_length != ULLONG_MAX);
+
+ /* The difference between advancing content_length and p is because
+ * the latter will automaticaly advance on the next loop iteration.
+ * Further, if content_length ends up at 0, we want to see the last
+ * byte again for our message complete callback.
+ */
+ MARK(body);
+ parser->content_length -= to_read;
+ p += to_read - 1;
+
+ if (parser->content_length == 0) {
+ UPDATE_STATE(s_message_done);
+
+ /* Mimic CALLBACK_DATA_NOADVANCE() but with one extra byte.
+ *
+ * The alternative to doing this is to wait for the next byte to
+ * trigger the data callback, just as in every other case. The
+ * problem with this is that this makes it difficult for the test
+ * harness to distinguish between complete-on-EOF and
+ * complete-on-length. It's not clear that this distinction is
+ * important for applications, but let's keep it for now.
+ */
+ CALLBACK_DATA_(body, p - body_mark + 1, p - data);
+ REEXECUTE();
+ }
+
+ break;
+ }
+
+ /* read until EOF */
+ case s_body_identity_eof:
+ MARK(body);
+ p = data + len - 1;
+
+ break;
+
+ case s_message_done:
+ UPDATE_STATE(NEW_MESSAGE());
+ CALLBACK_NOTIFY(message_complete);
+ if (parser->upgrade) {
+ /* Exit, the rest of the message is in a different protocol. */
+ RETURN((p - data) + 1);
+ }
+ break;
+
+ case s_chunk_size_start:
+ {
+ assert(nread == 1);
+ assert(parser->flags & F_CHUNKED);
+
+ unhex_val = unhex[(unsigned char)ch];
+ if (UNLIKELY(unhex_val == -1)) {
+ SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
+ goto error;
+ }
+
+ parser->content_length = unhex_val;
+ UPDATE_STATE(s_chunk_size);
+ break;
+ }
+
+ case s_chunk_size:
+ {
+ uint64_t t;
+
+ assert(parser->flags & F_CHUNKED);
+
+ if (ch == CR) {
+ UPDATE_STATE(s_chunk_size_almost_done);
+ break;
+ }
+
+ unhex_val = unhex[(unsigned char)ch];
+
+ if (unhex_val == -1) {
+ if (ch == ';' || ch == ' ') {
+ UPDATE_STATE(s_chunk_parameters);
+ break;
+ }
+
+ SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
+ goto error;
+ }
+
+ t = parser->content_length;
+ t *= 16;
+ t += unhex_val;
+
+ /* Overflow? Test against a conservative limit for simplicity. */
+ if (UNLIKELY((ULLONG_MAX - 16) / 16 < parser->content_length)) {
+ SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
+ goto error;
+ }
+
+ parser->content_length = t;
+ break;
+ }
+
+ case s_chunk_parameters:
+ {
+ assert(parser->flags & F_CHUNKED);
+ /* just ignore this shit. TODO check for overflow */
+ if (ch == CR) {
+ UPDATE_STATE(s_chunk_size_almost_done);
+ break;
+ }
+ break;
+ }
+
+ case s_chunk_size_almost_done:
+ {
+ assert(parser->flags & F_CHUNKED);
+ STRICT_CHECK(ch != LF);
+
+ parser->nread = 0;
+ nread = 0;
+
+ if (parser->content_length == 0) {
+ parser->flags |= F_TRAILING;
+ UPDATE_STATE(s_header_field_start);
+ } else {
+ UPDATE_STATE(s_chunk_data);
+ }
+ CALLBACK_NOTIFY(chunk_header);
+ break;
+ }
+
+ case s_chunk_data:
+ {
+ uint64_t to_read = MIN(parser->content_length,
+ (uint64_t) ((data + len) - p));
+
+ assert(parser->flags & F_CHUNKED);
+ assert(parser->content_length != 0
+ && parser->content_length != ULLONG_MAX);
+
+ /* See the explanation in s_body_identity for why the content
+ * length and data pointers are managed this way.
+ */
+ MARK(body);
+ parser->content_length -= to_read;
+ p += to_read - 1;
+
+ if (parser->content_length == 0) {
+ UPDATE_STATE(s_chunk_data_almost_done);
+ }
+
+ break;
+ }
+
+ case s_chunk_data_almost_done:
+ assert(parser->flags & F_CHUNKED);
+ assert(parser->content_length == 0);
+ STRICT_CHECK(ch != CR);
+ UPDATE_STATE(s_chunk_data_done);
+ CALLBACK_DATA(body);
+ break;
+
+ case s_chunk_data_done:
+ assert(parser->flags & F_CHUNKED);
+ STRICT_CHECK(ch != LF);
+ parser->nread = 0;
+ nread = 0;
+ UPDATE_STATE(s_chunk_size_start);
+ CALLBACK_NOTIFY(chunk_complete);
+ break;
+
+ default:
+ assert(0 && "unhandled state");
+ SET_ERRNO(HPE_INVALID_INTERNAL_STATE);
+ goto error;
+ }
+ }
+
+ /* Run callbacks for any marks that we have leftover after we ran out of
+ * bytes. There should be at most one of these set, so it's OK to invoke
+ * them in series (unset marks will not result in callbacks).
+ *
+ * We use the NOADVANCE() variety of callbacks here because 'p' has already
+ * overflowed 'data' and this allows us to correct for the off-by-one that
+ * we'd otherwise have (since CALLBACK_DATA() is meant to be run with a 'p'
+ * value that's in-bounds).
+ */
+
+ assert(((header_field_mark ? 1 : 0) +
+ (header_value_mark ? 1 : 0) +
+ (url_mark ? 1 : 0) +
+ (body_mark ? 1 : 0) +
+ (status_mark ? 1 : 0)) <= 1);
+
+ CALLBACK_DATA_NOADVANCE(header_field);
+ CALLBACK_DATA_NOADVANCE(header_value);
+ CALLBACK_DATA_NOADVANCE(url);
+ CALLBACK_DATA_NOADVANCE(body);
+ CALLBACK_DATA_NOADVANCE(status);
+
+ RETURN(len);
+
+error:
+ if (HTTP_PARSER_ERRNO(parser) == HPE_OK) {
+ SET_ERRNO(HPE_UNKNOWN);
+ }
+
+ RETURN(p - data);
+}
+
+
+/* Does the parser need to see an EOF to find the end of the message? */
+int
+http_message_needs_eof (const http_parser *parser)
+{
+ if (parser->type == HTTP_REQUEST) {
+ return 0;
+ }
+
+ /* See RFC 2616 section 4.4 */
+ if (parser->status_code / 100 == 1 || /* 1xx e.g. Continue */
+ parser->status_code == 204 || /* No Content */
+ parser->status_code == 304 || /* Not Modified */
+ (parser->flags & F_SKIPBODY)) { /* response to a HEAD request */
+ return 0;
+ }
+
+ if ((parser->flags & F_CHUNKED) || parser->content_length != ULLONG_MAX) {
+ return 0;
+ }
+
+ return 1;
+}
+
+
+int
+http_should_keep_alive (const http_parser *parser)
+{
+ if (parser->http_major > 0 && parser->http_minor > 0) {
+ /* HTTP/1.1 */
+ if (parser->flags & F_CONNECTION_CLOSE) {
+ return 0;
+ }
+ } else {
+ /* HTTP/1.0 or earlier */
+ if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) {
+ return 0;
+ }
+ }
+
+ return !http_message_needs_eof(parser);
+}
+
+
+const char *
+http_method_str (enum http_method m)
+{
+ return ELEM_AT(method_strings, m, "<unknown>");
+}
+
+const char *
+http_status_str (enum http_status s)
+{
+ switch (s) {
+#define XX(num, name, string) case HTTP_STATUS_##name: return #string;
+ HTTP_STATUS_MAP(XX)
+#undef XX
+ default: return "<unknown>";
+ }
+}
+
+void
+http_parser_init (http_parser *parser, enum http_parser_type t)
+{
+ void *data = parser->data; /* preserve application data */
+ memset(parser, 0, sizeof(*parser));
+ parser->data = data;
+ parser->type = t;
+ parser->state = (t == HTTP_REQUEST ? s_start_req : (t == HTTP_RESPONSE ? s_start_res : s_start_req_or_res));
+ parser->http_errno = HPE_OK;
+}
+
+void
+http_parser_settings_init(http_parser_settings *settings)
+{
+ memset(settings, 0, sizeof(*settings));
+}
+
+const char *
+http_errno_name(enum http_errno err) {
+ assert(((size_t) err) < ARRAY_SIZE(http_strerror_tab));
+ return http_strerror_tab[err].name;
+}
+
+const char *
+http_errno_description(enum http_errno err) {
+ assert(((size_t) err) < ARRAY_SIZE(http_strerror_tab));
+ return http_strerror_tab[err].description;
+}
+
+static enum http_host_state
+http_parse_host_char(enum http_host_state s, const char ch) {
+ switch(s) {
+ case s_http_userinfo:
+ case s_http_userinfo_start:
+ if (ch == '@') {
+ return s_http_host_start;
+ }
+
+ if (IS_USERINFO_CHAR(ch)) {
+ return s_http_userinfo;
+ }
+ break;
+
+ case s_http_host_start:
+ if (ch == '[') {
+ return s_http_host_v6_start;
+ }
+
+ if (IS_HOST_CHAR(ch)) {
+ return s_http_host;
+ }
+
+ break;
+
+ case s_http_host:
+ if (IS_HOST_CHAR(ch)) {
+ return s_http_host;
+ }
+
+ /* fall through */
+ case s_http_host_v6_end:
+ if (ch == ':') {
+ return s_http_host_port_start;
+ }
+
+ break;
+
+ case s_http_host_v6:
+ if (ch == ']') {
+ return s_http_host_v6_end;
+ }
+
+ /* fall through */
+ case s_http_host_v6_start:
+ if (IS_HEX(ch) || ch == ':' || ch == '.') {
+ return s_http_host_v6;
+ }
+
+ if (s == s_http_host_v6 && ch == '%') {
+ return s_http_host_v6_zone_start;
+ }
+ break;
+
+ case s_http_host_v6_zone:
+ if (ch == ']') {
+ return s_http_host_v6_end;
+ }
+
+ /* fall through */
+ case s_http_host_v6_zone_start:
+ /* RFC 6874 Zone ID consists of 1*( unreserved / pct-encoded) */
+ if (IS_ALPHANUM(ch) || ch == '%' || ch == '.' || ch == '-' || ch == '_' ||
+ ch == '~') {
+ return s_http_host_v6_zone;
+ }
+ break;
+
+ case s_http_host_port:
+ case s_http_host_port_start:
+ if (IS_NUM(ch)) {
+ return s_http_host_port;
+ }
+
+ break;
+
+ default:
+ break;
+ }
+ return s_http_host_dead;
+}
+
+static int
+http_parse_host(const char * buf, struct http_parser_url *u, int found_at) {
+ enum http_host_state s;
+
+ const char *p;
+ size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len;
+
+ assert(u->field_set & (1 << UF_HOST));
+
+ u->field_data[UF_HOST].len = 0;
+
+ s = found_at ? s_http_userinfo_start : s_http_host_start;
+
+ for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) {
+ enum http_host_state new_s = http_parse_host_char(s, *p);
+
+ if (new_s == s_http_host_dead) {
+ return 1;
+ }
+
+ switch(new_s) {
+ case s_http_host:
+ if (s != s_http_host) {
+ u->field_data[UF_HOST].off = (uint16_t)(p - buf);
+ }
+ u->field_data[UF_HOST].len++;
+ break;
+
+ case s_http_host_v6:
+ if (s != s_http_host_v6) {
+ u->field_data[UF_HOST].off = (uint16_t)(p - buf);
+ }
+ u->field_data[UF_HOST].len++;
+ break;
+
+ case s_http_host_v6_zone_start:
+ case s_http_host_v6_zone:
+ u->field_data[UF_HOST].len++;
+ break;
+
+ case s_http_host_port:
+ if (s != s_http_host_port) {
+ u->field_data[UF_PORT].off = (uint16_t)(p - buf);
+ u->field_data[UF_PORT].len = 0;
+ u->field_set |= (1 << UF_PORT);
+ }
+ u->field_data[UF_PORT].len++;
+ break;
+
+ case s_http_userinfo:
+ if (s != s_http_userinfo) {
+ u->field_data[UF_USERINFO].off = (uint16_t)(p - buf);
+ u->field_data[UF_USERINFO].len = 0;
+ u->field_set |= (1 << UF_USERINFO);
+ }
+ u->field_data[UF_USERINFO].len++;
+ break;
+
+ default:
+ break;
+ }
+ s = new_s;
+ }
+
+ /* Make sure we don't end somewhere unexpected */
+ switch (s) {
+ case s_http_host_start:
+ case s_http_host_v6_start:
+ case s_http_host_v6:
+ case s_http_host_v6_zone_start:
+ case s_http_host_v6_zone:
+ case s_http_host_port_start:
+ case s_http_userinfo:
+ case s_http_userinfo_start:
+ return 1;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+void
+http_parser_url_init(struct http_parser_url *u) {
+ memset(u, 0, sizeof(*u));
+}
+
+int
+http_parser_parse_url(const char *buf, size_t buflen, int is_connect,
+ struct http_parser_url *u)
+{
+ enum state s;
+ const char *p;
+ enum http_parser_url_fields uf, old_uf;
+ int found_at = 0;
+
+ if (buflen == 0) {
+ return 1;
+ }
+
+ u->port = u->field_set = 0;
+ s = is_connect ? s_req_server_start : s_req_spaces_before_url;
+ old_uf = UF_MAX;
+
+ for (p = buf; p < buf + buflen; p++) {
+ s = parse_url_char(s, *p);
+
+ /* Figure out the next field that we're operating on */
+ switch (s) {
+ case s_dead:
+ return 1;
+
+ /* Skip delimiters */
+ case s_req_schema_slash:
+ case s_req_schema_slash_slash:
+ case s_req_server_start:
+ case s_req_query_string_start:
+ case s_req_fragment_start:
+ continue;
+
+ case s_req_schema:
+ uf = UF_SCHEMA;
+ break;
+
+ case s_req_server_with_at:
+ found_at = 1;
+
+ /* fall through */
+ case s_req_server:
+ uf = UF_HOST;
+ break;
+
+ case s_req_path:
+ uf = UF_PATH;
+ break;
+
+ case s_req_query_string:
+ uf = UF_QUERY;
+ break;
+
+ case s_req_fragment:
+ uf = UF_FRAGMENT;
+ break;
+
+ default:
+ assert(!"Unexpected state");
+ return 1;
+ }
+
+ /* Nothing's changed; soldier on */
+ if (uf == old_uf) {
+ u->field_data[uf].len++;
+ continue;
+ }
+
+ u->field_data[uf].off = (uint16_t)(p - buf);
+ u->field_data[uf].len = 1;
+
+ u->field_set |= (1 << uf);
+ old_uf = uf;
+ }
+
+ /* host must be present if there is a schema */
+ /* parsing http:///toto will fail */
+ if ((u->field_set & (1 << UF_SCHEMA)) &&
+ (u->field_set & (1 << UF_HOST)) == 0) {
+ return 1;
+ }
+
+ if (u->field_set & (1 << UF_HOST)) {
+ if (http_parse_host(buf, u, found_at) != 0) {
+ return 1;
+ }
+ }
+
+ /* CONNECT requests can only contain "hostname:port" */
+ if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) {
+ return 1;
+ }
+
+ if (u->field_set & (1 << UF_PORT)) {
+ uint16_t off;
+ uint16_t len;
+ const char* p;
+ const char* end;
+ unsigned long v;
+
+ off = u->field_data[UF_PORT].off;
+ len = u->field_data[UF_PORT].len;
+ end = buf + off + len;
+
+ /* NOTE: The characters are already validated and are in the [0-9] range */
+ assert(off + len <= buflen && "Port number overflow");
+ v = 0;
+ for (p = buf + off; p < end; p++) {
+ v *= 10;
+ v += *p - '0';
+
+ /* Ports have a max value of 2^16 */
+ if (v > 0xffff) {
+ return 1;
+ }
+ }
+
+ u->port = (uint16_t) v;
+ }
+
+ return 0;
+}
+
+void
+http_parser_pause(http_parser *parser, int paused) {
+ /* Users should only be pausing/unpausing a parser that is not in an error
+ * state. In non-debug builds, there's not much that we can do about this
+ * other than ignore it.
+ */
+ if (HTTP_PARSER_ERRNO(parser) == HPE_OK ||
+ HTTP_PARSER_ERRNO(parser) == HPE_PAUSED) {
+ uint32_t nread = parser->nread; /* used by the SET_ERRNO macro */
+ SET_ERRNO((paused) ? HPE_PAUSED : HPE_OK);
+ } else {
+ assert(0 && "Attempting to pause parser in error state");
+ }
+}
+
+int
+http_body_is_final(const struct http_parser *parser) {
+ return parser->state == s_message_done;
+}
+
+unsigned long
+http_parser_version(void) {
+ return HTTP_PARSER_VERSION_MAJOR * 0x10000 |
+ HTTP_PARSER_VERSION_MINOR * 0x00100 |
+ HTTP_PARSER_VERSION_PATCH * 0x00001;
+}
+
+void
+http_parser_set_max_header_size(uint32_t size) {
+ max_header_size = size;
+}
--- /dev/null
+/*
+ * llist.c - Linked List implementation.
+ */
+#include <stddef.h>
+#include <stdlib.h>
+#include <llist/llist.h>
+
+static llist_node_t * LLIST_node_new(void * data);
+
+/*
+ * Returns NULL in case an error has occurred.
+ */
+llist_t * llist_new(void)
+{
+ llist_t * llist = malloc(sizeof(llist_t));
+ if (llist == NULL)
+ {
+ return NULL;
+ }
+ llist->len = 0;
+ llist->first = NULL;
+ llist->last = NULL;
+ return llist;
+}
+
+/*
+ * Destroys the linked list and calls a call-back function on each item.
+ * The result of the call back function will be ignored.
+ */
+void llist_free_cb(llist_t * llist, llist_cb cb, void * args)
+{
+ llist_node_t * node = llist->first;
+ llist_node_t * next;
+
+ while (node != NULL)
+ {
+ cb(node->data, args);
+ next = node->next;
+ free(node);
+ node = next;
+ }
+ free(llist);
+}
+
+/*
+ * Destroys the linked list and calls a call-back function on each item.
+ * The result of the call back function will be ignored.
+ */
+void llist_destroy(llist_t * llist, llist_destroy_cb cb)
+{
+ llist_node_t * node = llist->first;
+ llist_node_t * next;
+
+ while (node != NULL)
+ {
+ cb(node->data);
+ next = node->next;
+ free(node);
+ node = next;
+ }
+ free(llist);
+}
+
+/*
+ * Appends to the end of the list.
+ *
+ * Returns 0 if successful or -1 in case an error occurred.
+ * (in case of an error, the list remains unchanged)
+ */
+int llist_append(llist_t * llist, void * data)
+{
+ llist_node_t * node = LLIST_node_new(data);
+ if (node == NULL)
+ {
+ return -1;
+ }
+
+ llist->len++;
+ if (llist->last == NULL)
+ {
+ llist->last = node;
+ llist->first = llist->last;
+ }
+ else
+ {
+ llist->last->next = node;
+ llist->last = llist->last->next;
+ }
+
+ return 0;
+}
+
+/*
+ * Walk through the list. Call-back function will be called on each item and
+ * the sum of all results will be returned.
+ */
+int llist_walk(llist_t * llist, llist_cb cb, void * args)
+{
+ llist_node_t * node = llist->first;
+ int rc = 0;
+ while (node != NULL)
+ {
+ rc += cb(node->data, args);
+ node = node->next;
+ }
+ return rc;
+}
+
+/*
+ * Walk through the list. Call-back function will be called on each item and
+ * 'n' will be decremented by one for each non-zero call-back result.
+ * The walk will stop either on the end of the list or when 'n' is zero.
+ */
+void llist_walkn(llist_t * llist, size_t * n, llist_cb cb, void * args)
+{
+ llist_node_t * node = llist->first;
+ while (node != NULL && *n)
+ {
+ if (cb(node->data, args))
+ {
+ (*n)--;
+ }
+ node = node->next;
+ }
+}
+
+/*
+ * Remove and return the first item where 'cb' is not zero
+ * or NULL if not found unless 'cb' is NULL, then simple the
+ * data is compared to 'args'.
+ */
+void * llist_remove(llist_t * llist, llist_cb cb, void * args)
+{
+ llist_node_t * node = llist->first;
+ llist_node_t * prev = NULL;
+ void * data;
+
+ while (node != NULL)
+ {
+ if ((cb == NULL) ? node->data == args : cb(node->data, args))
+ {
+ if (prev == NULL)
+ {
+ /* this is the first node */
+ if (llist->last == llist->first)
+ {
+ /* this is the only node */
+ llist->first = llist->last = NULL;
+ }
+ else
+ {
+ llist->first = node->next;
+ }
+ }
+ else
+ {
+ prev->next = node->next;
+ if (prev->next == NULL)
+ {
+ /* this is the last node */
+ llist->last = prev;
+ }
+ }
+
+ data = node->data;
+ free(node);
+ llist->len--;
+
+ return data;
+ }
+ prev = node;
+ node = node->next;
+ }
+ return NULL;
+}
+
+/*
+ * Remove and return the last item in the list or NULL if empty.
+ */
+void * llist_pop(llist_t * llist)
+{
+ llist_node_t * node = llist->first;
+ llist_node_t * prev = NULL;
+ void * data = NULL;
+
+ if (node != NULL)
+ {
+ /* get the last node */
+ while (node->next != NULL)
+ {
+ prev = node;
+ node = node->next;
+ }
+
+ if (prev == NULL)
+ {
+ /* this is the only node */
+ llist->first = llist->last = NULL;
+ }
+ else
+ {
+ prev->next = NULL;
+ llist->last = prev;
+ }
+ data = node->data;
+ free(node);
+ llist->len--;
+ }
+
+ return data;
+}
+
+/*
+ * Remove and return the first item in the list or NULL if empty.
+ */
+void * llist_shift(llist_t * llist)
+{
+ llist_node_t * node = llist->first;
+ void * data = NULL;
+
+ if (node != NULL)
+ {
+ if (llist->last == llist->first)
+ {
+ llist->first = llist->last = NULL;
+ }
+ else
+ {
+ llist->first = node->next;
+ }
+
+ data = node->data;
+ free(node);
+ llist->len--;
+ }
+
+ return data;
+}
+
+/*
+ * Return the first item where 'cb' is not zero or NULL if not found.
+ */
+void * llist_get(llist_t * llist, llist_cb cb, void * args)
+{
+ llist_node_t * node = llist->first;
+ while (node != NULL)
+ {
+ if (cb(node->data, args))
+ {
+ return node->data;
+ }
+ node = node->next;
+ }
+ return NULL;
+}
+
+/*
+ * Copy the linked list to a simple list.
+ * (returns NULL in case an error has occurred)
+ */
+vec_t * llist2vec(llist_t * llist)
+{
+ vec_t * vec = vec_new(llist->len);
+
+ if (vec == NULL)
+ {
+ return NULL;
+ }
+
+ llist_node_t * node = llist->first;
+ size_t n;
+
+ for (n = 0; node != NULL; n++, node = node->next)
+ {
+ vec->data[n] = node->data;
+ }
+
+ vec->len = n;
+
+ return vec;
+}
+
+/*
+ * Returns NULL in case an error has occurred.
+ */
+static llist_node_t * LLIST_node_new(void * data)
+{
+ llist_node_t * llist_node = malloc(sizeof(llist_node_t));
+ if (llist_node == NULL)
+ {
+ return NULL;
+ }
+ llist_node->data = data;
+ llist_node->next = NULL;
+
+ return llist_node;
+}
+
--- /dev/null
+/*
+ * lock.c - Lock a directory by using a lock file.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <lock/lock.h>
+#include <xpath/xpath.h>
+#include <stddef.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef __APPLE__
+#include <libproc.h>
+#endif
+
+static lock_t LOCK_create(const char * lock_fn, pid_t pid);
+static int LOCK_get_process_name(char ** name, pid_t pid);
+
+/*
+ * Returns LOCK_NEW if a new lock is created, LOCK_OVERWRITE if an existing
+ * lock file was overwritten or a negative result in case of an error.
+ *
+ * Flags:
+ *
+ * LOCK_QUIT_IF_EXIST: do not check the lock file but simply consider the
+ * database locked if a lock file exists in the path.
+ */
+lock_t lock_lock(const char * path, int flags)
+{
+ pid_t pid = getpid();
+ pid_t ppid = 0;
+ char * lock_fn;
+ char * proc_name = NULL;
+ char * pproc_name = NULL;
+ char strpid[10];
+ int is_locked;
+ FILE * fp;
+ lock_t lock_rc;
+
+ /* initialize strpid to zero */
+ memset(&strpid, 0, sizeof(strpid));
+
+ if (asprintf(&lock_fn, "%s%s", path, ".lock") < 0)
+ {
+ return LOCK_MEM_ALLOC_ERR;
+ }
+
+ fp = fopen(lock_fn, "r");
+
+ if (fp != NULL)
+ {
+ if (flags & LOCK_QUIT_IF_EXIST)
+ {
+ fclose(fp);
+ free(lock_fn);
+ return LOCK_IS_LOCKED_ERR;
+ }
+
+ if (fread(strpid, sizeof(char), 9, fp))
+ {
+ ppid = strtoul(strpid, NULL, 10);
+ }
+ fclose(fp);
+
+ if (!ppid)
+ {
+ free(lock_fn);
+ return LOCK_READ_ERR;
+ }
+
+ if (ppid == pid)
+ {
+ /*
+ * The process id in the existing lock file is the same as 'this'
+ * process, we therefore can overwrite the lock.
+ */
+ lock_rc = LOCK_create(lock_fn, pid);
+ free(lock_fn);
+ return lock_rc;
+ }
+
+ if (LOCK_get_process_name(&proc_name, pid))
+ {
+ free(lock_fn);
+ return LOCK_MEM_ALLOC_ERR;
+ }
+
+ if (proc_name == NULL)
+ {
+ free(lock_fn);
+ return LOCK_PROCESS_NAME_ERR;
+ }
+
+ if (LOCK_get_process_name(&pproc_name, ppid))
+ {
+ free(proc_name);
+ free(lock_fn);
+ return LOCK_MEM_ALLOC_ERR;
+ }
+
+ /*
+ * If the process id in the lock file does not exist or if the
+ * process id is in use by another program than siridb we can
+ * assume the lock is not valid.
+ */
+ is_locked = (pproc_name != NULL && strcmp(proc_name, pproc_name) == 0);
+
+ free(proc_name);
+ free(pproc_name);
+
+ if (is_locked)
+ {
+ free(lock_fn);
+ return LOCK_IS_LOCKED_ERR;
+ }
+ }
+
+ lock_rc = LOCK_create(lock_fn, pid);
+ free(lock_fn);
+
+ return lock_rc;
+}
+
+/*
+ * Returns LOCK_REMOVED if successful or something else in case of an error.
+ */
+lock_t lock_unlock(const char * path)
+{
+ char * lock_fn;
+ lock_t lock_rc;
+
+ if (asprintf(&lock_fn, "%s%s", path, ".lock") < 0)
+ {
+ return LOCK_MEM_ALLOC_ERR;
+ }
+
+ lock_rc = (unlink(lock_fn)) ? LOCK_UNLINK_ERR : LOCK_REMOVED;
+ free(lock_fn);
+
+ return lock_rc;
+}
+
+/*
+ * Returns a message by lock result code.
+ */
+const char * lock_str(lock_t rc)
+{
+ switch (rc)
+ {
+ case LOCK_IS_LOCKED_ERR:
+ return "Database is locked by another process";
+ case LOCK_PROCESS_NAME_ERR:
+ return "Error occurred while reading process name";
+ case LOCK_WRITE_ERR:
+ return "Error occurred while writing the lock file";
+ case LOCK_READ_ERR:
+ return "Error occurred while reading existing lock file";
+ case LOCK_UNLINK_ERR:
+ return "Error occurred while removing the lock file";
+ case LOCK_MEM_ALLOC_ERR:
+ return "Memory allocation error occurred";
+ case LOCK_REMOVED:
+ return "Database is successfully unlocked";
+ case LOCK_NEW:
+ return "Database is successfully locked with a new lock file";
+ case LOCK_OVERWRITE:
+ return
+ "Database is successfully locked but a lock file existed which "
+ "indicates that the database was not closed correctly last time";
+ }
+ return "Unknown type";
+}
+
+/*
+ * Returns LOCK_NEW if a new lock is created, LOCK_OVERWRITE if an existing
+ * lock file was overwritten or LOCK_WRITE_ERR in case of an error.
+ */
+static lock_t LOCK_create(const char * lock_fn, pid_t pid)
+{
+ lock_t rc_success = (!xpath_file_exist(lock_fn)) ?
+ LOCK_NEW : LOCK_OVERWRITE;
+
+ FILE * fp = fopen(lock_fn, "w");
+ if (fp == NULL)
+ {
+ return LOCK_WRITE_ERR;
+ }
+ if (fprintf(fp, "%d", pid) < 0)
+ {
+ fclose(fp);
+ return LOCK_WRITE_ERR;
+ }
+ return (fclose(fp)) ? LOCK_WRITE_ERR : rc_success;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of a memory allocation error.
+ *
+ * When successful, name contains the process name by process id or NULL
+ * if the process is not found.
+ *
+ * In case 'name' is not NULL, 'calloc' is used to set the value and should
+ * be destroyed using free(). When -1 is returned 'name' is always NULL.
+ */
+#ifdef __APPLE__
+static int LOCK_get_process_name(char ** name, pid_t pid)
+{
+ struct proc_bsdinfo proc;
+
+ int st = proc_pidinfo(
+ pid,
+ PROC_PIDTBSDINFO,
+ 0,
+ &proc,
+ PROC_PIDTBSDINFO_SIZE);
+
+ if (st == 0)
+ {
+ return 0;
+ }
+ else if (st != PROC_PIDTBSDINFO_SIZE)
+ {
+ return -1;
+ }
+
+ *name = strdup(proc.pbi_comm);
+
+ if (*name == NULL)
+ {
+ return -1;
+ }
+
+ return 0;
+}
+#else
+static int LOCK_get_process_name(char ** name, pid_t pid)
+{
+ size_t n = 1024;
+ *name = (char *) calloc(n, sizeof(char));
+
+ if (*name == NULL)
+ {
+ return -1;
+ }
+ else
+ {
+ sprintf(*name, "/proc/%d/comm", pid);
+ FILE * fp = fopen(*name, "r");
+ if (fp == NULL)
+ {
+ /* most likely the file does not exist */
+ free(*name);
+ *name = NULL;
+ }
+ else
+ {
+ ssize_t size = getline(name, &n, fp);
+ fclose(fp);
+ if(size > 0)
+ {
+ if((*name)[size - 1] == '\n')
+ {
+ (*name)[size - 1] = 0;
+ }
+ } else if (size < 0)
+ {
+ free(*name);
+ *name = NULL;
+ return -1;
+ }
+ }
+ }
+ return 0;
+}
+#endif
--- /dev/null
+/*
+ * logger.h - Logging module.
+ */
+#include <logger/logger.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <time.h>
+
+logger_t Logger = {
+ .level=2,
+ .level_name=NULL,
+ .ostream=NULL,
+ .flags=0
+};
+
+#define LOGGER_CHR_MAP "DIWECU"
+
+#define KNRM "\x1B[0m" /* normal */
+#define KRED "\x1B[31m" /* error */
+#define KGRN "\x1B[32m" /* info */
+#define KYEL "\x1B[33m" /* warning */
+#define KBLU "\x1B[34m" /* -- not used -- */
+#define KMAG "\x1B[35m" /* critical */
+#define KCYN "\x1B[36m" /* debug */
+#define KWHT "\x1B[37m" /* -- not used -- */
+
+const char * LOGGER_LEVEL_NAMES[LOGGER_NUM_LEVELS] =
+ {"debug", "info", "warning", "error", "critical"};
+
+const char * LOGGER_COLOR_MAP[LOGGER_NUM_LEVELS] =
+ {KCYN, KGRN, KYEL, KRED, KMAG};
+
+#define LOGGER_LOG_STUFF(LEVEL) \
+{ \
+ time_t t = time(NULL); \
+ struct tm tm = *localtime(&t); \
+ if (Logger.flags & LOGGER_FLAG_COLORED) \
+ { \
+ fprintf(Logger.ostream, \
+ "%s[%c %d-%0*d-%0*d %0*d:%0*d:%0*d]" KNRM " ", \
+ LOGGER_COLOR_MAP[LEVEL], \
+ LOGGER_CHR_MAP[LEVEL], \
+ tm.tm_year + 1900, \
+ 2, tm.tm_mon + 1, \
+ 2, tm.tm_mday, \
+ 2, tm.tm_hour, \
+ 2, tm.tm_min, \
+ 2, tm.tm_sec); \
+ } \
+ else \
+ { \
+ fprintf(Logger.ostream, \
+ "[%c %d-%0*d-%0*d %0*d:%0*d:%0*d] ", \
+ LOGGER_CHR_MAP[LEVEL], \
+ tm.tm_year + 1900, \
+ 2, tm.tm_mon + 1, \
+ 2, tm.tm_mday, \
+ 2, tm.tm_hour, \
+ 2, tm.tm_min, \
+ 2, tm.tm_sec); \
+ } \
+ /* print the actual log line */ \
+ va_list args; \
+ va_start(args, fmt); \
+ vfprintf(Logger.ostream, fmt, args); \
+ va_end(args); \
+ /* write end of line and flush the stream */ \
+ fputc('\n', Logger.ostream); \
+ fflush(Logger.ostream); \
+}
+
+/*
+ * Initialize the Logger.
+ */
+void logger_init(LOGGER_IO_FILE * ostream, int log_level)
+{
+ Logger.ostream = ostream;
+ logger_set_level(log_level);
+}
+
+/*
+ * Set the logger to a given level. (name will be set too)
+ */
+void logger_set_level(int log_level)
+{
+ Logger.level = log_level;
+ Logger.level_name = LOGGER_LEVEL_NAMES[log_level];
+}
+
+/*
+ * Returns a log level name for a given log level.
+ */
+const char * logger_level_name(int log_level)
+{
+ return LOGGER_LEVEL_NAMES[log_level];
+}
+
+void log__debug(const char * fmt, ...)
+ LOGGER_LOG_STUFF(LOGGER_DEBUG)
+
+void log__info(const char * fmt, ...)
+ LOGGER_LOG_STUFF(LOGGER_INFO)
+
+void log__warning(const char * fmt, ...)
+ LOGGER_LOG_STUFF(LOGGER_WARNING)
+
+void log__error(const char * fmt, ...)
+ LOGGER_LOG_STUFF(LOGGER_ERROR)
+
+void log__critical(const char * fmt, ...)
+ LOGGER_LOG_STUFF(LOGGER_CRITICAL)
+
--- /dev/null
+/*
+ * util/omap.h
+ */
+#include <assert.h>
+#include <stdlib.h>
+#include <omap/omap.h>
+
+static void * omap__rm(omap_t * omap, omap__t ** omap_);
+static omap__t * omap__new(uint64_t id, void * data, omap__t * next);
+
+omap_t * omap_create(void)
+{
+ omap_t * omap = malloc(sizeof(omap_t));
+ if (!omap)
+ {
+ return NULL;
+ }
+
+ omap->next_ = NULL;
+ omap->n = 0;
+
+ return omap;
+}
+
+void omap_destroy(omap_t * omap, omap_destroy_cb cb)
+{
+ if (!omap)
+ {
+ return;
+ }
+ omap__t * cur = (omap__t *) omap;
+ omap__t * tmp;
+
+ for (; (tmp = cur->next_); cur = tmp)
+ {
+ if (cb && tmp)
+ {
+ (*cb)(tmp->data_);
+ }
+ free(cur);
+ }
+ free(cur);
+}
+
+/*
+ * In case of a duplicate id the return value is OMAP_ERR_EXIST and data
+ * will NOT be overwritten. On success the return value is OMAP_SUCCESS and
+ * if a memory error has occurred the return value is OMAP_ERR_ALLOC.
+ */
+int omap_add(omap_t * omap, uint64_t id, void * data)
+{
+ assert (omap);
+ assert (data);
+ omap__t * cur, * tmp;
+
+ for ( cur = (omap__t *) omap;
+ cur->next_ && cur->next_->id_ < id;
+ cur = cur->next_);
+
+ if (cur->next_ && cur->next_->id_ == id)
+ {
+ return OMAP_ERR_EXIST;
+ }
+
+ tmp = omap__new(id, data, cur->next_);
+ if (!tmp)
+ {
+ return OMAP_ERR_ALLOC;
+ }
+
+ omap->n++;
+ cur->next_ = tmp;
+
+ return OMAP_SUCCESS;
+}
+
+/*
+ * In case of a duplicate id the return value is the previous value and data
+ * will be overwritten. On success the return value is equal to void*data and
+ * if a memory error has occurred the return value is NULL.
+ */
+void * omap_set(omap_t * omap, uint64_t id, void * data)
+{
+ assert (omap);
+ assert (data);
+ omap__t * cur, * tmp;
+
+ for ( cur = (omap__t *) omap;
+ cur->next_ && cur->next_->id_ < id;
+ cur = cur->next_);
+
+ if (cur->next_ && cur->next_->id_ == id)
+ {
+ void * prev = cur->next_->data_;
+ cur->next_->data_ = data;
+ return prev;
+ }
+
+ tmp = omap__new(id, data, cur->next_);
+ if (!tmp)
+ return NULL;
+
+ omap->n++;
+ cur->next_ = tmp;
+
+ return data;
+}
+
+void * omap_get(omap_t * omap, uint64_t id)
+{
+ omap__t * cur = (omap__t *) omap;
+ while ((cur = cur->next_) && cur->id_ < id);
+
+ return cur && cur->id_ == id ? cur->data_ : NULL;
+}
+
+void * omap_rm(omap_t * omap, uint64_t id)
+{
+ omap__t * cur, * prev = (omap__t *) omap;
+ while ((cur = prev->next_) && cur->id_ < id)
+ {
+ prev = cur;
+ }
+
+ return cur && cur->id_ == id ? omap__rm(omap, &prev->next_) : NULL;
+}
+
+static void * omap__rm(omap_t * omap, omap__t ** omap_)
+{
+ omap__t * cur = *omap_;
+ void * data = cur->data_;
+ *omap_ = cur->next_;
+
+ free(cur);
+ --omap->n;
+
+ return data;
+}
+
+static omap__t * omap__new(uint64_t id, void * data, omap__t * next)
+{
+ omap__t * omap = malloc(sizeof(omap__t));
+ if (!omap)
+ return NULL;
+
+ omap->id_ = id;
+ omap->data_ = data;
+ omap->next_ = next;
+
+ return omap;
+}
--- /dev/null
+/*
+ * owcrypt.c - One Way Encryption. (used for storing a database user password)
+ *
+ * purpose:
+ * - provide an encryption algorithm to prevent password guessing.
+ * - passwords should be stored using one way encryption so the original
+ * is lost.
+ *
+ */
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <owcrypt/owcrypt.h>
+
+#define OLD_SALT_SZ 8
+
+#define CRYPTSET(x) t = encrypted[x] + k; encrypted[x] = VCHARS[t % 64]
+
+#define VCHARS "./0123456789" \
+ "abcdefghijklmnopqrstuvwxyz" \
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+static const int P[109] = {
+ 281, 283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367,
+ 373, 379, 383, 389, 397, 401, 409, 419, 421, 431, 433, 439, 443, 449,
+ 457, 461, 463, 467, 479, 487, 491, 499, 503, 509, 521, 523, 541, 547,
+ 557, 563, 569, 571, 577, 587, 593, 599, 601, 607, 613, 617, 619, 631,
+ 641, 643, 647, 653, 659, 661, 673, 677, 683, 691, 701, 709, 719, 727,
+ 733, 739, 743, 751, 757, 761, 769, 773, 787, 797, 809, 811, 821, 823,
+ 827, 829, 839, 853, 857, 859, 863, 877, 881, 883, 887, 907, 911, 919,
+ 929, 937, 941, 947, 953, 967, 971, 977, 983, 991, 997 };
+
+static const unsigned int END = OWCRYPT_SZ - 1;
+
+static void owcrypt0(
+ const char * password,
+ const char * salt,
+ char * encrypted);
+static void owcrypt1(
+ const char * password,
+ const char * salt,
+ char * encrypted);
+
+
+/*
+ * Encrypts a password using a salt.
+ *
+ * Usage:
+ *
+ * char[OWCRYPT_SZ] encrypted;
+ * owcrypt("my_password", "saltsalt$1", encrypted);
+ *
+ * # Checking can be done like this:
+ * char[OWCRYPT_SZ] pw;
+ * owcrypt("pasword_to_check", encrypted, pw");
+ * if (strcmp(pw, encrypted) == 0) {
+ * # valid
+ * }
+ *
+ * Parameters:
+ * const char * password: must be a terminated string
+ * const char * salt: must be a string with at least a length OWCRYPT_SALT_SZ.
+ * char * encrypted: must be able to hold at least OWCRYPT_SZ chars.
+ */
+void owcrypt(const char * password, const char * salt, char * encrypted)
+{
+ switch (salt[OWCRYPT_SALT_SZ - 1])
+ {
+ case '0':
+ /* deprecated version */
+ owcrypt0(password, salt, encrypted);
+ break;
+ case '1':
+ owcrypt1(password, salt, encrypted);
+ break;
+ default:
+ encrypted[0] = '\0';
+ break;
+ }
+}
+
+/*
+ * Generate a random salt.
+ *
+ * Make sure random is initialized, for example using:
+ * srand(time(NULL));
+ */
+void owcrypt_gen_salt(char * salt)
+{
+ int i;
+ for (i = 0; i < OWCRYPT_SALT_SZ - 2; i++)
+ {
+ salt[i] = VCHARS[rand() % 64];
+ }
+ salt[OWCRYPT_SALT_SZ - 2] = '$';
+ salt[OWCRYPT_SALT_SZ - 1] = '1';
+}
+
+static void owcrypt1(
+ const char * password,
+ const char * salt,
+ char * encrypted)
+{
+ unsigned int i, j, c;
+ unsigned long int k, t;
+ unsigned const char * p, * w;
+
+ memset(encrypted, 0, OWCRYPT_SZ);
+
+ for (i = 0; i < OWCRYPT_SALT_SZ; i++)
+ {
+ encrypted[i] = salt[i];
+ }
+
+ for ( w = (unsigned char *) password, i = OWCRYPT_SALT_SZ;
+ i < END;
+ i++, w++)
+ {
+ if (!*w)
+ {
+ w = (unsigned char *) password;
+ }
+
+ for (k = 0, p = (unsigned char *) password; *p; p++)
+ {
+ for (c = 0; c < OWCRYPT_SALT_SZ; c++)
+ {
+ k += P[(salt[c] + *p + *w + i + k) % 109];
+ }
+ }
+ CRYPTSET(i);
+
+ for (j = k % 3 + OWCRYPT_SALT_SZ, c = k % 5 + 1; j < END; j += c)
+ {
+ CRYPTSET(j);
+ }
+ }
+}
+
+/* deprecated */
+static void owcrypt0(
+ const char * password,
+ const char * salt,
+ char * encrypted)
+{
+ unsigned int i, c, j;
+ unsigned long long k;
+ const char * p, * w;
+
+ for (i = 0; i < OLD_SALT_SZ; i++)
+ {
+ encrypted[i] = salt[i];
+ }
+
+ encrypted[OLD_SALT_SZ] = '$';
+ encrypted[OLD_SALT_SZ + 1] = '0';
+
+ for (w = password, i = OLD_SALT_SZ + 2; i < OWCRYPT_SZ - 1; i++, w++)
+ {
+ if (!*w)
+ {
+ w = password;
+ }
+
+ for (k = 0, p = password; *p; p++)
+ {
+ for (c = 0; c < OLD_SALT_SZ; c++)
+ {
+ j = salt[c] + *p + *w;
+ k += j * P[(j + i + k) % 109];
+ }
+ }
+ encrypted[i] = VCHARS[k % 64];
+ }
+ encrypted[OWCRYPT_SZ - 1] = '\0';
+}
--- /dev/null
+/*
+ * procinfo.c - Process info for the current running process.
+ *
+ * Got most information from:
+ *
+ * http://stackoverflow.com/questions/63166/how-to-determine-cpu-and-
+ * memory-consumption-from-inside-a-process
+ *
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <limits.h>
+#include <logger/logger.h>
+#include <procinfo/procinfo.h>
+#include <inttypes.h>
+
+#ifdef __APPLE__
+#include <mach/task.h>
+#include <mach/mach_init.h>
+#include <mach/task_info.h>
+#include <libproc.h>
+#else
+#include <xpath/xpath.h>
+#endif
+
+
+static long int parse_line(char * line);
+
+/*
+ * Unused function.
+ */
+long int procinfo_total_virtual_memory(void)
+{
+ /* Value is returned in KB */
+ int64_t result = -1;
+ FILE * file = fopen("/proc/self/status", "r");
+
+ if (file != NULL)
+ {
+ char line[128];
+
+ while (fgets(line, 128, file) != NULL)
+ {
+ if (strncmp(line, "VmSize:", 7) == 0)
+ {
+ result = parse_line(line);
+ break;
+ }
+ }
+ fclose(file);
+ }
+
+ return result;
+}
+
+#ifdef __APPLE__
+long int procinfo_total_physical_memory(void)
+{
+ kern_return_t ret;
+ mach_msg_type_number_t out_count;
+ mach_task_basic_info_data_t taskinfo;
+
+ taskinfo.virtual_size = 0;
+ out_count = MACH_TASK_BASIC_INFO_COUNT;
+
+ ret = task_info(
+ mach_task_self(),
+ MACH_TASK_BASIC_INFO,
+ (task_info_t) &taskinfo,
+ &out_count);
+
+ return (ret == KERN_SUCCESS) ?
+ (long int) taskinfo.resident_size / 1024 : -1;
+}
+#else
+long int procinfo_total_physical_memory(void)
+{
+ /* Value is returned in KB */
+ long int result = -1;
+ FILE * file = fopen("/proc/self/status", "r");
+
+ if (file != NULL)
+ {
+ char line[128];
+
+ while (fgets(line, 128, file) != NULL)
+ {
+ if (strncmp(line, "VmRSS:", 6) == 0)
+ {
+ result = parse_line(line);
+ break;
+ }
+ }
+ fclose(file);
+ }
+ return result;
+}
+#endif
+
+#ifdef __APPLE__
+long int procinfo_open_files(const char * path, int include_fd)
+{
+ pid_t pid = getpid();
+ size_t len = strlen(path);
+ long int buffer_size = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, 0, 0);
+ long int buffer_used, number_of_proc_fds, i, count;
+
+ if (buffer_size < 0)
+ {
+ return -1;
+ }
+
+ struct proc_fdinfo * fd_info = malloc(buffer_size);
+
+ if (fd_info == NULL)
+ {
+ return -1;
+ }
+
+ buffer_used = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, fd_info, buffer_size);
+ number_of_proc_fds = buffer_used / PROC_PIDLISTFD_SIZE;
+
+ for (i = 0, count = 0; i < number_of_proc_fds; i++)
+ {
+ if (fd_info[i].proc_fdtype == PROX_FDTYPE_VNODE)
+ {
+ struct vnode_fdinfowithpath vnode_info;
+
+ int res = proc_pidfdinfo(
+ pid,
+ fd_info[i].proc_fd,
+ PROC_PIDFDVNODEPATHINFO,
+ &vnode_info,
+ PROC_PIDFDVNODEPATHINFO_SIZE);
+
+ if ( res == PROC_PIDFDVNODEPATHINFO_SIZE &&
+ strncmp(path, vnode_info.pvip.vip_path, len) == 0)
+ {
+ count++;
+ }
+ else if (
+ res == PROC_PIDFDVNODEPATHINFO_SIZE &&
+ include_fd >= 0 &&
+ include_fd == fd_info[i].proc_fd)
+ {
+ include_fd = -1;
+ count++;
+ };
+
+ }
+ }
+ free(fd_info);
+ return count;
+}
+#else
+long int procinfo_open_files(const char * path, int include_fd)
+{
+ long int count = 0;
+ DIR * dirp;
+ struct dirent * entry;
+ size_t len = strlen(path);
+ char buffer[XPATH_MAX];
+ char buf[XPATH_MAX];
+
+ if ((dirp = opendir("/proc/self/fd")) == NULL)
+ {
+ return -1;
+ }
+
+ while ((entry = readdir(dirp)) != NULL)
+ {
+ if (entry->d_type == DT_REG || entry->d_type == DT_LNK)
+ {
+ if (snprintf(
+ buffer,
+ XPATH_MAX,
+ "/proc/self/fd/%s",
+ entry->d_name) >= XPATH_MAX)
+ {
+ buffer[XPATH_MAX-1] = '\0';
+ }
+ if (realpath(buffer, buf) == NULL)
+ {
+ continue;
+ }
+
+ if (strncmp(path, buf, len) == 0)
+ {
+ count++;
+ }
+ else if (
+ include_fd >= 0 &&
+ include_fd == strtol(entry->d_name, NULL, 10))
+ {
+ include_fd = -1;
+ count++;
+ };
+ }
+ }
+ closedir(dirp);
+
+ return count;
+}
+#endif
+
+static long int parse_line(char * line)
+{
+ long int i = strlen(line);
+
+ while (*line < '0' || *line > '9')
+ {
+ line++;
+ }
+
+ line[i - 3] = '\0';
+
+ i = atol(line);
+
+ return i;
+}
--- /dev/null
+/*
+ * qpack.c - Efficient binary serialization format.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <qpack/qpack.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <logger/logger.h>
+#include <assert.h>
+#include <siri/err.h>
+
+
+#define QPACK_MAX_FMT_SIZE 1024
+
+#define QP_RESIZE(LEN) \
+if (packer->len + LEN > packer->buffer_size) \
+{ \
+ packer->buffer_size = \
+ ((packer->len + LEN) / packer->alloc_size + 1) \
+ * packer->alloc_size; \
+ unsigned char * tmp = (unsigned char *) realloc( \
+ packer->buffer, packer->buffer_size); \
+ if (tmp == NULL) \
+ { \
+ ERR_ALLOC \
+ packer->buffer_size = packer->len; \
+ return -1; \
+ } \
+ packer->buffer = tmp; \
+}
+
+#define QP_PLAIN_OBJ(QP_TYPE) \
+{ \
+ QP_RESIZE(1) \
+ packer->buffer[packer->len++] = QP_TYPE; \
+ return 0; \
+}
+
+#define QP_PREPARE_RAW \
+ QP_RESIZE((9 + len)) \
+ if (len < 100) \
+ { \
+ packer->buffer[packer->len++] = 128 + len; \
+ } \
+ else if (len <= UINT8_MAX) \
+ { \
+ uint8_t length = (uint8_t) len; \
+ packer->buffer[packer->len++] = QP_RAW8; \
+ packer->buffer[packer->len++] = length; \
+ } \
+ else if (len <= UINT16_MAX) \
+ { \
+ uint16_t length = (uint16_t) len; \
+ packer->buffer[packer->len++] = QP_RAW16; \
+ memcpy(packer->buffer + packer->len, &length, 2); \
+ packer->len += 2; \
+ } \
+ else if (len <= UINT32_MAX) \
+ { \
+ uint32_t length = (uint32_t) len; \
+ packer->buffer[packer->len++] = QP_RAW32; \
+ memcpy(packer->buffer + packer->len, &length, 4); \
+ packer->len += 4; \
+ } \
+ else \
+ { \
+ packer->buffer[packer->len++] = QP_RAW64; \
+ memcpy(packer->buffer + packer->len, &len, 8); \
+ packer->len += 8; \
+ }
+
+#define QP_UNPACK_RAW(uintx_t) \
+{ \
+ QP_UNPACK_CHECK_SZ(sizeof(uintx_t)) \
+ size_t sz = (size_t) *((uintx_t *) unpacker->pt); \
+ unpacker->pt += sizeof(uintx_t); \
+ QP_UNPACK_CHECK_SZ(sz) \
+ if (qp_obj != NULL) \
+ { \
+ qp_obj->tp = QP_RAW; \
+ qp_obj->via.raw = unpacker->pt; \
+ qp_obj->len = sz; \
+ } \
+ unpacker->pt += sz; \
+ return QP_RAW; \
+}
+
+#define QP_UNPACK_INT(intx_t) \
+{ \
+ QP_UNPACK_CHECK_SZ(sizeof(intx_t)) \
+ int64_t val = (int64_t) *((intx_t *) unpacker->pt); \
+ if (qp_obj != NULL) \
+ { \
+ qp_obj->tp = QP_INT64; \
+ qp_obj->via.int64 = val; \
+ } \
+ unpacker->pt += sizeof(intx_t); \
+ return QP_INT64; \
+}
+
+#define QP_UNPACK_CHECK_SZ(size) \
+if (unpacker->pt + size > unpacker->end) \
+{ \
+ if (qp_obj != NULL) \
+ { \
+ qp_obj->tp = QP_ERR; \
+ } \
+ return QP_ERR; \
+}
+
+static qp_types_t QP_print_unpacker(
+ qp_types_t tp,
+ qp_unpacker_t * unpacker,
+ qp_obj_t * qp_obj);
+
+/*
+ * Initialize unpacker object.
+ */
+void qp_unpacker_init(qp_unpacker_t * unpacker, unsigned char * pt, size_t len)
+{
+ unpacker->source = NULL;
+ unpacker->pt = pt;
+ unpacker->end = pt + len;
+}
+
+/*
+ * Destroy unpacker object. (parsing NULL is not allowed)
+ */
+void qp_unpacker_ff_free(qp_unpacker_t * unpacker)
+{
+ assert(unpacker != NULL);
+ free(unpacker->source);
+ free(unpacker);
+}
+
+/*
+ * Returns a unpacker object or NULL in case an error occurred. The error
+ * message will logged using at least log_error().
+ *
+ * In case and only in case of a memory (malloc) error, a signal will be raised.
+ */
+qp_unpacker_t * qp_unpacker_ff(const char * fn)
+{
+ FILE * fp;
+ ssize_t size;
+ qp_unpacker_t * unpacker;
+
+ fp = fopen(fn, "r");
+ if (fp == NULL)
+ {
+ log_error("Could not read '%s'", fn);
+ return NULL;
+ }
+
+ /* get the size */
+ if (
+ fseeko(fp, 0, SEEK_END) ||
+ (size = ftello(fp)) == -1 ||
+ fseeko(fp, 0, SEEK_SET))
+ {
+ log_error("Cannot not read size of file '%s'", fn);
+ unpacker = NULL;
+ }
+ else
+ {
+ unpacker = malloc(sizeof(qp_unpacker_t));
+ if (unpacker == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ unpacker->source = malloc(size);
+ if (unpacker->source == NULL)
+ {
+ ERR_ALLOC
+ qp_unpacker_ff_free(unpacker);
+ unpacker = NULL;
+ }
+ else if (fread(unpacker->source, size, 1, fp) == 1)
+ {
+ unpacker->pt = unpacker->source;
+ unpacker->end = unpacker->source + size;
+ }
+ else
+ {
+ log_error("Cannot not read from file '%s'", fn);
+ qp_unpacker_ff_free(unpacker);
+ unpacker = NULL;
+ }
+ }
+ }
+
+ if (fclose(fp))
+ {
+ /*
+ * Not critical and should probably never happen because file was
+ * open only for reading.
+ */
+ log_error("Could not close file: '%s'", fn);
+ }
+
+ return unpacker;
+}
+
+/*
+ * Returns a new packer object or NULL in case of an error.
+ */
+qp_packer_t * qp_packer_new(size_t alloc_size)
+{
+ qp_packer_t * packer = malloc(sizeof(qp_packer_t));
+ if (packer != NULL)
+ {
+ packer->alloc_size = alloc_size;
+ packer->buffer_size = packer->alloc_size;
+ packer->len = 0;
+
+ packer->buffer = malloc(packer->buffer_size);
+ if (packer->buffer == NULL)
+ {
+ free(packer);
+ packer = NULL;
+ }
+ }
+ return packer;
+}
+
+/*
+ * Destroy packer object. (parsing NULL is not allowed)
+ */
+void qp_packer_free(qp_packer_t * packer)
+{
+ assert(packer != NULL);
+ free(packer->buffer);
+ free(packer);
+}
+
+/*
+ * Extend packer with another packer (source).
+ *
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_packer_extend(qp_packer_t * packer, qp_packer_t * source)
+{
+ QP_RESIZE(source->len)
+ memcpy(packer->buffer + packer->len, source->buffer, source->len);
+ packer->len += source->len;
+ return 0;
+}
+
+/*
+ * Extend packer with data from an unpacker.
+ * (only the object at the current position will be copied)
+ *
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_packer_extend_fu(qp_packer_t * packer, qp_unpacker_t * unpacker)
+{
+ /* mark the start of the object */
+ const unsigned char * start = unpacker->pt;
+
+ /* jump to the end of the current object */
+ qp_skip_next(unpacker);
+
+ /* get size of the total object */
+ size_t size = unpacker->pt - start;
+
+ /* write object to the packer */
+ QP_RESIZE(size)
+ memcpy(packer->buffer + packer->len, start, size);
+ packer->len += size;
+ return 0;
+}
+
+/*
+ * Print qpack content.
+ */
+void qp_print(unsigned char * pt, size_t len)
+{
+ qp_obj_t qp_obj;
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pt, len);
+ QP_print_unpacker(qp_next(&unpacker, &qp_obj), &unpacker, &qp_obj);
+ printf("\n");
+}
+
+/*
+ * This is like qp_next(unpacker, NULL) but in case of a map or array the
+ * total object is skipped. The return type can be used to check what the
+ * skipped object was.
+ */
+qp_types_t qp_skip_next(qp_unpacker_t * unpacker)
+{
+ qp_types_t tp = qp_next(unpacker, NULL);
+ int count;
+ switch (tp)
+ {
+
+ case QP_ARRAY0:
+ case QP_ARRAY1:
+ case QP_ARRAY2:
+ case QP_ARRAY3:
+ case QP_ARRAY4:
+ case QP_ARRAY5:
+ count = tp - QP_ARRAY0;
+ while (count--)
+ {
+ qp_skip_next(unpacker);
+ }
+ return tp;
+ case QP_MAP0:
+ case QP_MAP1:
+ case QP_MAP2:
+ case QP_MAP3:
+ case QP_MAP4:
+ case QP_MAP5:
+ count = (tp - QP_MAP0) * 2;
+ while (count--)
+ {
+ qp_skip_next(unpacker);
+ }
+ return tp;
+ case QP_ARRAY_OPEN:
+ while (tp && tp != QP_ARRAY_CLOSE)
+ {
+ tp = qp_skip_next(unpacker);
+ }
+ return QP_ARRAY_OPEN;
+ case QP_MAP_OPEN:
+ /* read first key or end or close */
+ tp = qp_skip_next(unpacker);
+ while (tp && tp != QP_MAP_CLOSE)
+ {
+ /* read value */
+ qp_skip_next(unpacker);
+
+ /* read next key or end or close */
+ tp = qp_skip_next(unpacker);
+ }
+ return QP_MAP_OPEN;
+ default:
+ return tp;
+ }
+}
+
+/*
+ * This function will not add more than QPACK_MAX_FMT_SIZE and will still
+ * return 0 in case longer strings are parsed.
+ *
+ * Use qp_add_fmt_safe() in case you want to add longer or unknown length.
+ *
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_fmt(qp_packer_t * packer, const char * fmt, ...)
+{
+ va_list args;
+ char buffer[QPACK_MAX_FMT_SIZE];
+ va_start(args, fmt);
+ if (vsnprintf(buffer, QPACK_MAX_FMT_SIZE, fmt, args) >= QPACK_MAX_FMT_SIZE)
+ {
+ buffer[QPACK_MAX_FMT_SIZE-1] = '\0';
+ }
+ va_end(args);
+ return qp_add_string(packer, buffer);
+}
+
+/*
+ * Like qp_add_fmt() but works for any length.
+ *
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_fmt_safe(qp_packer_t * packer, const char * fmt, ...)
+{
+ int rc;
+ va_list args;
+ char * buffer;
+
+ va_start(args, fmt);
+ rc = vasprintf(&buffer, fmt, args);
+ va_end(args);
+
+ if (rc == -1)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ rc = qp_add_string(packer, buffer);
+ free(buffer);
+ return rc;
+}
+
+/*
+ * Adds a raw string to the packer fixed to len chars.
+ *
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_raw(qp_packer_t * packer, const unsigned char * raw, size_t len)
+{
+ QP_PREPARE_RAW
+ memcpy(packer->buffer + packer->len, raw, len);
+ packer->len += len;
+ return 0;
+}
+
+/* shortcuts for qp_add_raw() */
+int qp_add_string(qp_packer_t * packer, const char * str)
+{
+ return qp_add_raw(packer, (unsigned char *) str, strlen(str));
+}
+int qp_add_string_term(qp_packer_t * packer, const char * str)
+{
+ return qp_add_raw(packer, (unsigned char *) str, strlen(str) + 1);
+}
+int qp_add_string_term_n(qp_packer_t * packer, const char * str, size_t n)
+{
+ return qp_add_raw(packer, (unsigned char *) str, n + 1);
+}
+
+/*
+ * Adds a raw string to the packer and appends a terminator (0) so the written
+ * length is len + 1
+ *
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_raw_term(qp_packer_t * packer, const unsigned char * raw, size_t len_raw)
+{
+ /* add one because 0 (terminator) should be included with the length */
+ size_t len = len_raw + 1;
+ QP_PREPARE_RAW
+
+ /* now take 1 because we want to copy one less than the new len */
+ memcpy(packer->buffer + packer->len, raw, len_raw);
+ packer->len += len;
+ packer->buffer[packer->len - 1] = '\0';
+ return 0;
+}
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_double(qp_packer_t * packer, double real)
+{
+ QP_RESIZE(9)
+ if (real == 0.0)
+ {
+ packer->buffer[packer->len++] = QP_DOUBLE_0;
+ }
+ else if (real == 1.0)
+ {
+ packer->buffer[packer->len++] = QP_DOUBLE_1;
+ }
+ else if (real == -1.0)
+ {
+ packer->buffer[packer->len++] = QP_DOUBLE_N1;
+ }
+ else
+ {
+ packer->buffer[packer->len++] = QP_DOUBLE;
+ memcpy(packer->buffer + packer->len, &real, sizeof(double));
+ packer->len += sizeof(double);
+ }
+ return 0;
+}
+
+
+
+
+
+
+
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_int64(qp_packer_t * packer, int64_t integer)
+{
+ int8_t i8;
+ if ((i8 = (int8_t) integer) == integer)
+ {
+ QP_RESIZE(2)
+ if (i8 >= 0 && i8 < 64)
+ {
+ packer->buffer[packer->len++] = i8;
+ }
+ else if (i8 >= -60 && i8 < 0)
+ {
+ packer->buffer[packer->len++] = 63 - i8;
+ }
+ else
+ {
+ packer->buffer[packer->len++] = QP_INT8;
+ packer->buffer[packer->len++] = i8;
+ }
+ return 0;
+ }
+
+ int16_t i16;
+ if ((i16 = (int16_t) integer) == integer)
+ {
+ QP_RESIZE(3)
+ packer->buffer[packer->len++] = QP_INT16;
+ memcpy(packer->buffer + packer->len, &i16, sizeof(int16_t));
+ packer->len += sizeof(int16_t);
+ return 0;
+ }
+
+ int32_t i32;
+ if ((i32 = (int32_t) integer) == integer)
+ {
+ QP_RESIZE(5)
+ packer->buffer[packer->len++] = QP_INT32;
+ memcpy(packer->buffer + packer->len, &i32, sizeof(int32_t));
+ packer->len += sizeof(int32_t);
+ return 0;
+ }
+
+
+ QP_RESIZE(9)
+ packer->buffer[packer->len++] = QP_INT64;
+ memcpy(packer->buffer + packer->len, &integer, sizeof(int64_t));
+ packer->len += sizeof(int64_t);
+ return 0;
+}
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_true(qp_packer_t * packer) QP_PLAIN_OBJ(QP_TRUE)
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_false(qp_packer_t * packer) QP_PLAIN_OBJ(QP_FALSE)
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_null(qp_packer_t * packer) QP_PLAIN_OBJ(QP_NULL)
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int qp_add_type(qp_packer_t * packer, qp_types_t tp)
+{
+ assert(tp >= QP_ARRAY0 && tp <= QP_MAP_CLOSE);
+ QP_RESIZE(1)
+ packer->buffer[packer->len++] = tp;
+ return 0;
+}
+
+/*
+ * Returns 0 if successful and EOF in case an error occurred.
+ */
+int qp_fadd_type(qp_fpacker_t * fpacker, qp_types_t tp)
+{
+ assert(tp >= QP_ARRAY0 && tp <= QP_MAP_CLOSE);
+ return (fputc(tp, fpacker) == (int) tp) ? 0 : EOF;
+}
+
+/*
+ * Returns 0 if successful and EOF in case an error occurred.
+ */
+int qp_fadd_raw(qp_fpacker_t * fpacker, const unsigned char * raw, size_t len)
+{
+ if (len < 100)
+ {
+ if (fputc(128 + len, fpacker) == EOF)
+ {
+ return EOF;
+ }
+ }
+ else if (len <= UINT8_MAX)
+ {
+ if ( fputc(QP_RAW8, fpacker) == EOF ||
+ fputc(len, fpacker) == EOF)
+ {
+ return EOF;
+ }
+ }
+ else if (len <= UINT16_MAX)
+ {
+ if ( fputc(QP_RAW16, fpacker) == EOF ||
+ fwrite(&len, sizeof(uint16_t), 1, fpacker) != 1)
+ {
+ return EOF;
+ }
+ }
+ else if (len <= UINT32_MAX)
+ {
+ if ( fputc(QP_RAW32, fpacker) == EOF ||
+ fwrite(&len, sizeof(uint32_t), 1, fpacker) != 1)
+ {
+ return EOF;
+ }
+ }
+ else
+ {
+ if ( fputc(QP_RAW64, fpacker) == EOF ||
+ fwrite(&len, sizeof(uint64_t), 1, fpacker) != 1)
+ {
+ return EOF;
+ }
+ }
+ return (!len || (fwrite(raw, len, 1, fpacker) == 1)) ? 0 : EOF;
+}
+
+/*
+ * Returns 0 if successful and EOF in case an error occurred.
+ */
+int qp_fadd_string(qp_fpacker_t * fpacker, const char * str)
+{
+ return qp_fadd_raw(fpacker, (unsigned char *) str, strlen(str));
+}
+
+/*
+ * Returns 0 if successful and EOF in case an error occurred.
+ */
+int qp_fadd_int64(qp_fpacker_t * fpacker, int64_t integer)
+{
+ int8_t i8;
+ if ((i8 = (int8_t) integer) == integer)
+ {
+ if (i8 >= 0 && i8 < 64)
+ {
+ return (fputc(i8, fpacker) != EOF) ? 0 : EOF;
+ }
+ if (i8 >= -60 && i8 < 0)
+ {
+ return (fputc(63 - i8, fpacker) != EOF) ? 0 : EOF;
+ }
+ return (fputc(QP_INT8, fpacker) != EOF && fputc(i8, fpacker) != EOF)
+ ? 0 : EOF;
+ }
+ int16_t i16;
+ if ((i16 = (int16_t) integer) == integer)
+ {
+ return (fputc(QP_INT16, fpacker) != EOF &&
+ fwrite(&i16, sizeof(int16_t), 1, fpacker) == 1) ? 0 : EOF;
+ }
+
+ int32_t i32;
+ if ((i32 = (int32_t) integer) == integer)
+ {
+ return (fputc(QP_INT32, fpacker) != EOF &&
+ fwrite(&i32, sizeof(int32_t), 1, fpacker) == 1) ? 0 : EOF;
+ }
+
+ return (fputc(QP_INT64, fpacker) != EOF &&
+ fwrite(&integer, sizeof(int64_t), 1, fpacker) == 1) ? 0 : EOF;
+}
+
+/*
+ * Returns 0 if successful and EOF in case an error occurred.
+ */
+int qp_fadd_double(qp_fpacker_t * fpacker, double real)
+{
+ if (real == 0.0)
+ {
+ return (fputc(QP_DOUBLE_0, fpacker) != EOF) ? 0 : EOF;
+ }
+ else if (real == 1.0)
+ {
+ return (fputc(QP_DOUBLE_1, fpacker) != EOF) ? 0 : EOF;
+ }
+ else if (real == -1.0)
+ {
+ return (fputc(QP_DOUBLE_N1, fpacker) != EOF) ? 0 : EOF;
+ }
+ else
+ {
+ return (fputc(QP_DOUBLE, fpacker) != EOF &&
+ fwrite(&real, sizeof(double), 1, fpacker) == 1) ? 0 : EOF;
+ }
+ return 0;
+}
+
+/*
+ * Jump to the next object. If 'qp_obj' is not NULL, the object will be stored
+ * in qp_obj so you can use it later.
+ *
+ * Returns one of the following: (these are the ONLY possible return values)
+ *
+ * QP_END, QP_RAW, QP_INT64, QP_DOUBLE
+ * QP_TRUE, QP_FALSE, QP_NULL, QP_ARRAY0..5, QP_MAP0..5,
+ * QP_ARRAY_OPEN, QP_ARRAY_CLOSE, QP_MAP_OPEN, QP_MAP_CLOSE
+ *
+ * Its fine to reuse the same object without calling free in between.
+ */
+qp_types_t qp_next(qp_unpacker_t * unpacker, qp_obj_t * qp_obj)
+{
+ uint8_t tp;
+
+ if (unpacker->pt >= unpacker->end)
+ {
+ if (qp_obj != NULL)
+ {
+ qp_obj->tp = QP_END;
+ }
+ return QP_END;
+ }
+
+ tp = *unpacker->pt;
+ unpacker->pt++;
+
+ switch(tp)
+ {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ case 8:
+ case 9:
+ case 10:
+ case 11:
+ case 12:
+ case 13:
+ case 14:
+ case 15:
+ case 16:
+ case 17:
+ case 18:
+ case 19:
+ case 20:
+ case 21:
+ case 22:
+ case 23:
+ case 24:
+ case 25:
+ case 26:
+ case 27:
+ case 28:
+ case 29:
+ case 30:
+ case 31:
+ case 32:
+ case 33:
+ case 34:
+ case 35:
+ case 36:
+ case 37:
+ case 38:
+ case 39:
+ case 40:
+ case 41:
+ case 42:
+ case 43:
+ case 44:
+ case 45:
+ case 46:
+ case 47:
+ case 48:
+ case 49:
+ case 50:
+ case 51:
+ case 52:
+ case 53:
+ case 54:
+ case 55:
+ case 56:
+ case 57:
+ case 58:
+ case 59:
+ case 60:
+ case 61:
+ case 62:
+ case 63:
+ if (qp_obj != NULL)
+ {
+ qp_obj->tp = QP_INT64;
+ qp_obj->via.int64 = (int64_t) tp;
+ }
+ return QP_INT64;
+
+ case 64:
+ case 65:
+ case 66:
+ case 67:
+ case 68:
+ case 69:
+ case 70:
+ case 71:
+ case 72:
+ case 73:
+ case 74:
+ case 75:
+ case 76:
+ case 77:
+ case 78:
+ case 79:
+ case 80:
+ case 81:
+ case 82:
+ case 83:
+ case 84:
+ case 85:
+ case 86:
+ case 87:
+ case 88:
+ case 89:
+ case 90:
+ case 91:
+ case 92:
+ case 93:
+ case 94:
+ case 95:
+ case 96:
+ case 97:
+ case 98:
+ case 99:
+ case 100:
+ case 101:
+ case 102:
+ case 103:
+ case 104:
+ case 105:
+ case 106:
+ case 107:
+ case 108:
+ case 109:
+ case 110:
+ case 111:
+ case 112:
+ case 113:
+ case 114:
+ case 115:
+ case 116:
+ case 117:
+ case 118:
+ case 119:
+ case 120:
+ case 121:
+ case 122:
+ case 123:
+ if (qp_obj != NULL)
+ {
+ qp_obj->tp = QP_INT64;
+ qp_obj->via.int64 = (int64_t) 63 - tp;
+ }
+ return QP_INT64;
+
+ case 124:
+ /* Object hooks are not supported yet */
+ if (qp_obj != NULL)
+ {
+ qp_obj->tp = QP_HOOK;
+ }
+ return QP_HOOK;
+
+ case 125:
+ case 126:
+ case 127:
+ /* unpack fixed doubles -1.0, 0.0 or 1.0 */
+ if (qp_obj != NULL)
+ {
+ qp_obj->tp = QP_DOUBLE;
+ qp_obj->via.real = (double) (tp - 126);
+ }
+ return QP_DOUBLE;
+
+ case 128:
+ case 129:
+ case 130:
+ case 131:
+ case 132:
+ case 133:
+ case 134:
+ case 135:
+ case 136:
+ case 137:
+ case 138:
+ case 139:
+ case 140:
+ case 141:
+ case 142:
+ case 143:
+ case 144:
+ case 145:
+ case 146:
+ case 147:
+ case 148:
+ case 149:
+ case 150:
+ case 151:
+ case 152:
+ case 153:
+ case 154:
+ case 155:
+ case 156:
+ case 157:
+ case 158:
+ case 159:
+ case 160:
+ case 161:
+ case 162:
+ case 163:
+ case 164:
+ case 165:
+ case 166:
+ case 167:
+ case 168:
+ case 169:
+ case 170:
+ case 171:
+ case 172:
+ case 173:
+ case 174:
+ case 175:
+ case 176:
+ case 177:
+ case 178:
+ case 179:
+ case 180:
+ case 181:
+ case 182:
+ case 183:
+ case 184:
+ case 185:
+ case 186:
+ case 187:
+ case 188:
+ case 189:
+ case 190:
+ case 191:
+ case 192:
+ case 193:
+ case 194:
+ case 195:
+ case 196:
+ case 197:
+ case 198:
+ case 199:
+ case 200:
+ case 201:
+ case 202:
+ case 203:
+ case 204:
+ case 205:
+ case 206:
+ case 207:
+ case 208:
+ case 209:
+ case 210:
+ case 211:
+ case 212:
+ case 213:
+ case 214:
+ case 215:
+ case 216:
+ case 217:
+ case 218:
+ case 219:
+ case 220:
+ case 221:
+ case 222:
+ case 223:
+ case 224:
+ case 225:
+ case 226:
+ case 227:
+ /* unpack fixed sized raw strings */
+ {
+ size_t size = tp - 128;
+ QP_UNPACK_CHECK_SZ(size)
+
+ if (qp_obj != NULL)
+ {
+ qp_obj->tp = QP_RAW;
+ qp_obj->via.raw = unpacker->pt;
+ qp_obj->len = size;
+ }
+ unpacker->pt += size;
+ return QP_RAW;
+ }
+ case 228:
+ QP_UNPACK_RAW(uint8_t)
+ case 229:
+ QP_UNPACK_RAW(uint16_t)
+ case 230:
+ QP_UNPACK_RAW(uint32_t)
+ case 231:
+ QP_UNPACK_RAW(uint64_t)
+
+ case 232:
+ QP_UNPACK_INT(int8_t)
+ case 233:
+ QP_UNPACK_INT(int16_t)
+ case 234:
+ QP_UNPACK_INT(int32_t)
+ case 235:
+ QP_UNPACK_INT(int64_t)
+
+ case 236:
+ QP_UNPACK_CHECK_SZ(sizeof(double))
+ if (qp_obj != NULL)
+ {
+ qp_obj->tp = QP_DOUBLE;
+ memcpy(&qp_obj->via.real, unpacker->pt, sizeof(double));
+ }
+ unpacker->pt += sizeof(double);
+ return QP_DOUBLE;
+
+ case 237:
+ case 238:
+ case 239:
+ case 240:
+ case 241:
+ case 242:
+ case 243:
+ case 244:
+ case 245:
+ case 246:
+ case 247:
+ case 248:
+ case 249:
+ case 250:
+ case 251:
+ case 252:
+ case 253:
+ case 254:
+ case 255:
+ if (qp_obj != NULL)
+ {
+ qp_obj->tp = tp;
+ }
+ return tp;
+ }
+
+ assert (0);
+ return -1;
+}
+
+/*
+ * Returns one of the following: (these are the ONLY possible return values)
+ *
+ * QP_END, QP_HOOK, QP_RAW, QP_INT64, QP_DOUBLE
+ * QP_TRUE, QP_FALSE, QP_NULL, QP_ARRAY0..5, QP_MAP0..5,
+ * QP_ARRAY_OPEN, QP_ARRAY_CLOSE, QP_MAP_OPEN, QP_MAP_CLOSE
+ */
+qp_types_t qp_current(qp_unpacker_t * unpacker)
+{
+ if (unpacker->pt >= unpacker->end)
+ {
+ return QP_END;
+ }
+
+ switch ((uint8_t) *unpacker->pt)
+ {
+ case 124:
+ return QP_HOOK;
+ case 125:
+ case 126:
+ case 127:
+ return QP_DOUBLE;
+ case 128:
+ case 129:
+ case 130:
+ case 131:
+ case 132:
+ case 133:
+ case 134:
+ case 135:
+ case 136:
+ case 137:
+ case 138:
+ case 139:
+ case 140:
+ case 141:
+ case 142:
+ case 143:
+ case 144:
+ case 145:
+ case 146:
+ case 147:
+ case 148:
+ case 149:
+ case 150:
+ case 151:
+ case 152:
+ case 153:
+ case 154:
+ case 155:
+ case 156:
+ case 157:
+ case 158:
+ case 159:
+ case 160:
+ case 161:
+ case 162:
+ case 163:
+ case 164:
+ case 165:
+ case 166:
+ case 167:
+ case 168:
+ case 169:
+ case 170:
+ case 171:
+ case 172:
+ case 173:
+ case 174:
+ case 175:
+ case 176:
+ case 177:
+ case 178:
+ case 179:
+ case 180:
+ case 181:
+ case 182:
+ case 183:
+ case 184:
+ case 185:
+ case 186:
+ case 187:
+ case 188:
+ case 189:
+ case 190:
+ case 191:
+ case 192:
+ case 193:
+ case 194:
+ case 195:
+ case 196:
+ case 197:
+ case 198:
+ case 199:
+ case 200:
+ case 201:
+ case 202:
+ case 203:
+ case 204:
+ case 205:
+ case 206:
+ case 207:
+ case 208:
+ case 209:
+ case 210:
+ case 211:
+ case 212:
+ case 213:
+ case 214:
+ case 215:
+ case 216:
+ case 217:
+ case 218:
+ case 219:
+ case 220:
+ case 221:
+ case 222:
+ case 223:
+ case 224:
+ case 225:
+ case 226:
+ case 227:
+ case 228:
+ case 229:
+ case 230:
+ case 231:
+ return QP_RAW;
+ case 232:
+ case 233:
+ case 234:
+ case 235:
+ return QP_INT64;
+ case 236:
+ return QP_DOUBLE;
+ case 237:
+ case 238:
+ case 239:
+ case 240:
+ case 241:
+ case 242:
+ case 243:
+ case 244:
+ case 245:
+ case 246:
+ case 247:
+ case 248:
+ case 249:
+ case 250:
+ case 251:
+ case 252:
+ case 253:
+ case 254:
+ case 255:
+ return (uint8_t) *unpacker->pt;
+ default:
+ return QP_INT64;
+ }
+}
+
+/*
+ * Recursive function for printing qpack data. (this function uses an unpacker
+ * object but its also used when using qp_packer_print().
+ */
+static qp_types_t QP_print_unpacker(
+ qp_types_t tp,
+ qp_unpacker_t * unpacker,
+ qp_obj_t * qp_obj)
+{
+ int count;
+ int found;
+ switch (tp)
+ {
+ case QP_INT64:
+ printf("%" PRId64, qp_obj->via.int64);
+ break;
+ case QP_DOUBLE:
+ printf("%f", qp_obj->via.real);
+ break;
+ case QP_RAW:
+ printf("\"%.*s\"", (int) qp_obj->len, qp_obj->via.raw);
+ break;
+ case QP_TRUE:
+ printf("true");
+ break;
+ case QP_FALSE:
+ printf("false");
+ break;
+ case QP_NULL:
+ printf("null");
+ break;
+ case QP_ARRAY0:
+ case QP_ARRAY1:
+ case QP_ARRAY2:
+ case QP_ARRAY3:
+ case QP_ARRAY4:
+ case QP_ARRAY5:
+ printf("[");
+ count = tp - QP_ARRAY0;
+ tp = qp_next(unpacker, qp_obj);
+ for (found = 0; count-- && tp; found = 1)
+ {
+ if (found )
+ {
+ printf(", ");
+ }
+ tp = QP_print_unpacker(tp, unpacker, qp_obj);
+ }
+ printf("]");
+ return tp;
+ case QP_MAP0:
+ case QP_MAP1:
+ case QP_MAP2:
+ case QP_MAP3:
+ case QP_MAP4:
+ case QP_MAP5:
+ printf("{");
+ count = tp - QP_MAP0;
+ tp = qp_next(unpacker, qp_obj);
+ for (found = 0; count-- && tp; found = 1)
+ {
+ if (found )
+ {
+ printf(", ");
+ }
+ tp = QP_print_unpacker(tp, unpacker, qp_obj);
+ printf(": ");
+ tp = QP_print_unpacker(tp, unpacker, qp_obj);
+ }
+ printf("}");
+ return tp;
+ case QP_ARRAY_OPEN:
+ printf("[");
+ tp = qp_next(unpacker, qp_obj);
+ for (count = 0; tp && tp != QP_ARRAY_CLOSE; count = 1)
+ {
+ if (count)
+ {
+ printf(", ");
+ }
+ tp = QP_print_unpacker(tp, unpacker, qp_obj);
+ }
+ printf("]");
+ break;
+ case QP_MAP_OPEN:
+ printf("{");
+ tp = qp_next(unpacker, qp_obj);
+ for (count = 0; tp && tp != QP_MAP_CLOSE; count = 1)
+ {
+ if (count)
+ {
+ printf(", ");
+ }
+ tp = QP_print_unpacker(tp, unpacker, qp_obj);
+ printf(": ");
+ tp = QP_print_unpacker(tp, unpacker, qp_obj);
+ }
+ printf("}");
+ break;
+ default:
+ break;
+ }
+ return qp_next(unpacker, qp_obj);
+}
--- /dev/null
+/*
+ * qpjson.c - Convert between QPack and JSON
+ */
+#include <assert.h>
+#include <qpjson/qpjson.h>
+#include <qpack/qpack.h>
+#include <logger/logger.h>
+
+static yajl_gen_status qp__to_json(yajl_gen g, qp_unpacker_t * up, qp_obj_t * obj)
+{
+ switch (obj->tp)
+ {
+ case QP_ERR:
+ case QP_ARRAY_CLOSE:
+ case QP_MAP_CLOSE:
+ case QP_END:
+ /* END, ARRAY_CLOSE and MAP_CLOSE should never occur since they can
+ * only happen after "*OPEN", and this case is handled in the *OPEN
+ * parts itself
+ */
+ return yajl_gen_in_error_state;
+ case QP_INT64:
+ return yajl_gen_integer(g, obj->via.int64);
+ case QP_DOUBLE:
+ return yajl_gen_double(g, obj->via.real);
+ case QP_RAW:
+ return yajl_gen_string(
+ g, (unsigned char *) obj->via.raw, obj->len);
+ case QP_TRUE:
+ return yajl_gen_bool(g, 1 /* true */);
+ case QP_FALSE:
+ return yajl_gen_bool(g, 0 /* false */);
+ case QP_NULL:
+ return yajl_gen_null(g);
+ case QP_ARRAY0:
+ case QP_ARRAY1:
+ case QP_ARRAY2:
+ case QP_ARRAY3:
+ case QP_ARRAY4:
+ case QP_ARRAY5:
+ {
+ size_t i = obj->tp - QP_ARRAY0;
+
+ yajl_gen_status stat;
+ if ((stat = yajl_gen_array_open(g)) != yajl_gen_status_ok)
+ return stat;
+
+ while (i--)
+ {
+ qp_next(up, obj);
+ if ((stat = qp__to_json(g, up, obj)) != yajl_gen_status_ok)
+ return stat;
+ }
+
+ return yajl_gen_array_close(g);
+ }
+ case QP_MAP0:
+ case QP_MAP1:
+ case QP_MAP2:
+ case QP_MAP3:
+ case QP_MAP4:
+ case QP_MAP5:
+ {
+ size_t i = obj->tp - QP_MAP0;
+ yajl_gen_status stat;
+ if ((stat = yajl_gen_map_open(g)) != yajl_gen_status_ok)
+ return stat;
+
+ while (i--)
+ {
+ qp_next(up, obj);
+ if ((stat = qp__to_json(g, up, obj)) != yajl_gen_status_ok)
+ return stat;
+
+ qp_next(up, obj);
+ if ((stat = qp__to_json(g, up, obj)) != yajl_gen_status_ok)
+ return stat;
+ }
+
+ return yajl_gen_map_close(g);
+ }
+ case QP_ARRAY_OPEN:
+ {
+ yajl_gen_status stat;
+ if ((stat = yajl_gen_array_open(g)) != yajl_gen_status_ok)
+ return stat;
+
+ while(qp_next(up, obj) && obj->tp != QP_ARRAY_CLOSE)
+ {
+ if ((stat = qp__to_json(g, up, obj)) != yajl_gen_status_ok)
+ return stat;
+ }
+
+ return yajl_gen_array_close(g);
+ }
+ case QP_MAP_OPEN:
+ {
+ yajl_gen_status stat;
+ if ((stat = yajl_gen_map_open(g)) != yajl_gen_status_ok)
+ return stat;
+
+ while(qp_next(up, obj) && obj->tp != QP_MAP_CLOSE)
+ {
+ if ((stat = qp__to_json(g, up, obj)) != yajl_gen_status_ok)
+ return stat;
+
+ qp_next(up, obj);
+ if ((stat = qp__to_json(g, up, obj)) != yajl_gen_status_ok)
+ return stat;
+ }
+ return yajl_gen_map_close(g);
+ }
+ }
+ return yajl_gen_in_error_state;
+}
+
+yajl_gen_status qpjson_qp_to_json(
+ void * src,
+ size_t src_n,
+ unsigned char ** dst,
+ size_t * dst_n,
+ int flags)
+{
+ qp_unpacker_t up;
+ qp_obj_t obj;
+ yajl_gen g;
+ yajl_gen_status stat;
+
+ if (src_n == 0)
+ {
+ *dst_n = 0;
+ *dst = NULL;
+ return yajl_gen_status_ok;
+ }
+
+ qp_unpacker_init(&up, src, src_n);
+
+ g = yajl_gen_alloc(NULL);
+ if (!g)
+ return yajl_gen_in_error_state;
+
+ yajl_gen_config(g, yajl_gen_beautify, flags & QPJSON_FLAG_BEAUTIFY);
+ yajl_gen_config(g, yajl_gen_validate_utf8, flags & QPJSON_FLAG_VALIDATE_UTF8);
+
+ qp_next(&up, &obj);
+
+ assert (obj.tp != QP_END);
+
+ if ((stat = qp__to_json(g, &up, &obj)) == yajl_gen_status_ok)
+ {
+ const unsigned char * tmp;
+ yajl_gen_get_buf(g, &tmp, dst_n);
+ *dst = malloc(*dst_n);
+ if (*dst)
+ memcpy(*dst, tmp, *dst_n);
+ else
+ stat = yajl_gen_in_error_state;
+ }
+
+ yajl_gen_free(g);
+ return stat;
+}
+
+static int reformat_null(void * ctx)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_null(pk);
+}
+
+static int reformat_boolean(void * ctx, int boolean)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == (boolean ? qp_add_true(pk) : qp_add_false(pk));
+}
+
+static int reformat_integer(void * ctx, long long i)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_int64(pk, i);
+}
+
+static int reformat_double(void * ctx, double d)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_double(pk, d);
+}
+
+static int reformat_string(void * ctx, const unsigned char * s, size_t n)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_raw(pk, s, n);
+}
+
+static int reformat_map_key(void * ctx, const unsigned char * s, size_t n)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_raw(pk, s, n);
+}
+
+static int reformat_start_map(void * ctx)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_type(pk, QP_MAP_OPEN);
+}
+
+
+static int reformat_end_map(void * ctx)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_type(pk, QP_MAP_CLOSE);
+}
+
+static int reformat_start_array(void * ctx)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_type(pk, QP_ARRAY_OPEN);
+}
+
+static int reformat_end_array(void * ctx)
+{
+ qp_packer_t * pk = (qp_packer_t *) ctx;
+ return 0 == qp_add_type(pk, QP_ARRAY_CLOSE);
+}
+
+static void take_buffer(
+ qp_packer_t * pk,
+ char ** dst,
+ size_t * dst_n)
+{
+ *dst = (char *) pk->buffer;
+ *dst_n = pk->len;
+ pk->buffer = NULL;
+ pk->buffer_size = 0;
+}
+
+static yajl_callbacks qpjson__callbacks = {
+ reformat_null,
+ reformat_boolean,
+ reformat_integer,
+ reformat_double,
+ NULL,
+ reformat_string,
+ reformat_start_map,
+ reformat_map_key,
+ reformat_end_map,
+ reformat_start_array,
+ reformat_end_array
+};
+
+yajl_status qpjson_json_to_qp(
+ const void * src,
+ size_t src_n,
+ char ** dst,
+ size_t * dst_n)
+{
+ yajl_handle hand;
+ yajl_status stat = yajl_status_error;
+ qp_packer_t * pk = qp_packer_new(src_n);
+
+ if (pk == NULL)
+ return stat;
+
+
+ hand = yajl_alloc(&qpjson__callbacks, NULL, pk);
+ if (!hand)
+ goto fail1;
+
+ stat = yajl_parse(hand, src, src_n);
+
+ if (stat == yajl_status_ok)
+ take_buffer(pk, dst, dst_n);
+
+ yajl_free(hand);
+
+fail1:
+ qp_packer_free(pk);
+ return stat;
+}
--- /dev/null
+/*
+ * api.c
+ */
+#include <string.h>
+#include <siri/api.h>
+#include <assert.h>
+#include <math.h>
+#include <siri/siri.h>
+#include <siri/db/users.h>
+#include <qpjson/qpjson.h>
+#include <siri/db/query.h>
+#include <siri/db/insert.h>
+#include <siri/service/account.h>
+#include <siri/net/tcp.h>
+
+#define API__HEADER_MAX_SZ 256
+
+static uv_tcp_t api__uv_server;
+static http_parser_settings api__settings;
+
+typedef struct
+{
+ char * query;
+ size_t query_n;
+ double factor;
+} api__query_t;
+
+#define API__ICMP_WITH(__s, __n, __w) \
+ (__n == strlen(__w) && strncasecmp(__s, __w, __n) == 0)
+
+#define API__CMP_WITH(__s, __n, __w) \
+ (__n == strlen(__w) && strncmp(__s, __w, __n) == 0)
+
+static const char api__content_type[3][20] = {
+ "text/plain",
+ "application/json",
+ "application/qpack",
+};
+
+static const char api__html_header[10][32] = {
+ "200 OK",
+ "400 Bad Request",
+ "401 Unauthorized",
+ "403 Forbidden",
+ "404 Not Found",
+ "405 Method Not Allowed",
+ "415 Unsupported Media Type",
+ "422 Unprocessable Entity",
+ "500 Internal Server Error",
+ "503 Service Unavailable",
+};
+
+static const char api__default_body[10][30] = {
+ "OK\r\n",
+ "BAD REQUEST\r\n",
+ "UNAUTHORIZED\r\n",
+ "FORBIDDEN\r\n",
+ "NOT FOUND\r\n",
+ "METHOD NOT ALLOWED\r\n",
+ "UNSUPPORTED MEDIA TYPE\r\n",
+ "UNPROCESSABLE ENTITY\r\n",
+ "INTERNAL SERVER ERROR\r\n",
+ "SERVICE UNAVAILABLE\r\n",
+};
+
+
+inline static int api__header(
+ char * ptr,
+ const siri_api_header_t ht,
+ const siri_api_content_t ct,
+ size_t content_length)
+{
+ int len = sprintf(
+ ptr,
+ "HTTP/1.1 %s\r\n" \
+ "Content-Type: %s\r\n" \
+ "Content-Length: %zu\r\n" \
+ "\r\n",
+ api__html_header[ht],
+ api__content_type[ct],
+ content_length);
+ return len;
+}
+
+static inline _Bool api__istarts_with(
+ const char ** str,
+ size_t * strn,
+ const char * with,
+ size_t withn)
+{
+ if (*strn < withn || strncasecmp(*str, with, withn))
+ {
+ return false;
+ }
+ *str += withn;
+ *strn -= withn;
+ return true;
+}
+
+static inline _Bool api__starts_with(
+ const char ** str,
+ size_t * strn,
+ const char * with,
+ size_t withn)
+{
+ if (*strn < withn || strncmp(*str, with, withn))
+ {
+ return false;
+ }
+ *str += withn;
+ *strn -= withn;
+ return true;
+}
+
+static void api__alloc_cb(
+ uv_handle_t * UNUSED_handle __attribute__((unused)),
+ size_t UNUSED_sugsz __attribute__((unused)),
+ uv_buf_t * buf)
+{
+ buf->base = malloc(HTTP_MAX_HEADER_SIZE);
+ buf->len = buf->base ? HTTP_MAX_HEADER_SIZE-1 : 0;
+}
+
+static void api__reset(siri_api_request_t * ar)
+{
+ /* Reset buffer in case multiple HTTP requests are used */
+ free (ar->buf);
+
+ if (ar->siridb)
+ {
+ siridb_decref(ar->siridb);
+ ar->siridb = NULL;
+ }
+
+ if (ar->origin)
+ {
+ siridb_user_decref((siridb_user_t *) ar->origin);
+ ar->origin = NULL;
+ }
+
+ ar->buf = NULL;
+ ar->len = 0;
+ ar->size = 0;
+ ar->on_state = NULL;
+ ar->service_authenticated = 0;
+ ar->request_type = SIRI_API_RT_NONE;
+ ar->content_type = SIRI_API_CT_TEXT;
+}
+
+static void api__data_cb(
+ uv_stream_t * uvstream,
+ ssize_t n,
+ const uv_buf_t * buf)
+{
+ size_t parsed;
+ siri_api_request_t * ar = uvstream->data;
+
+ if (!ar->ref)
+ goto done;
+
+ if (n < 0)
+ {
+ if (n != UV_EOF)
+ log_error(uv_strerror(n));
+
+ sirinet_stream_decref(ar);
+ goto done;
+ }
+
+ buf->base[HTTP_MAX_HEADER_SIZE-1] = '\0';
+
+ parsed = http_parser_execute(
+ &ar->parser,
+ &api__settings,
+ buf->base, n);
+
+ if (ar->parser.upgrade)
+ {
+ /* TODO: do we need to do something? */
+ log_debug("upgrade to a new protocol");
+ }
+ else if (parsed != (size_t) n)
+ {
+ log_warning("error parsing HTTP API request");
+ sirinet_stream_decref(ar);
+ }
+
+done:
+ free(buf->base);
+}
+
+static int api__headers_complete_cb(http_parser * parser)
+{
+ siri_api_request_t * ar = parser->data;
+
+ assert (!ar->buf);
+
+ if (parser->content_length != ULLONG_MAX)
+ {
+ ar->buf = malloc(parser->content_length);
+ if (ar->buf)
+ {
+ ar->len = parser->content_length;
+ }
+ }
+
+ return 0;
+}
+
+static void api__get_siridb(siri_api_request_t * ar, const char * at, size_t n)
+{
+ const char * pt = at;
+ size_t nn = 0;
+
+ while (n && *pt != '?')
+ {
+ ++pt;
+ --n;
+ ++nn;
+ }
+ ar->siridb = siridb_getn(siri.siridb_list, at, nn);
+ if (ar->siridb)
+ {
+ siridb_incref(ar->siridb);
+ }
+}
+
+static int api__url_cb(http_parser * parser, const char * at, size_t n)
+{
+ siri_api_request_t * ar = parser->data;
+
+ if (api__starts_with(&at, &n, "/query/", strlen("/query/")))
+ {
+ ar->request_type = SIRI_API_RT_QUERY;
+ api__get_siridb(ar, at, n);
+ }
+ else if (api__starts_with(&at, &n, "/insert/", strlen("/insert/")))
+ {
+ ar->request_type = SIRI_API_RT_INSERT;
+ api__get_siridb(ar, at, n);
+ }
+ else if (API__CMP_WITH(at, n, "/new-account"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_NEW_ACCOUNT;
+ }
+ else if (API__CMP_WITH(at, n, "/change-password"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_CHANGE_PASSWORD;
+ }
+ else if (API__CMP_WITH(at, n, "/drop-account"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_DROP_ACCOUNT;
+ }
+ else if (API__CMP_WITH(at, n, "/new-database"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_NEW_DATABASE;
+ }
+ else if (API__CMP_WITH(at, n, "/new-pool"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_NEW_POOL;
+ }
+ else if (API__CMP_WITH(at, n, "/new-replica"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_NEW_REPLICA;
+ }
+ else if (API__CMP_WITH(at, n, "/drop-database"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_DROP_DATABASE;
+ }
+ else if (API__CMP_WITH(at, n, "/get-version"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_GET_VERSION;
+ }
+ else if (API__CMP_WITH(at, n, "/get-accounts"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_GET_ACCOUNTS;
+ }
+ else if (API__CMP_WITH(at, n, "/get-databases"))
+ {
+ ar->request_type = SIRI_APT_RT_SERVICE;
+ ar->service_type = SERVICE_GET_DATABASES;
+ }
+ return 0;
+}
+
+static void api__connection_cb(uv_stream_t * server, int status)
+{
+ int rc;
+ siri_api_request_t * ar;
+
+ if (status < 0)
+ {
+ log_error("HTTP API connection error: `%s`", uv_strerror(status));
+ return;
+ }
+
+ log_debug("received a HTTP API call");
+
+ ar = calloc(1, sizeof(siri_api_request_t));
+ if (!ar)
+ {
+ ERR_ALLOC
+ return;
+ }
+
+ ar->stream = malloc(sizeof(uv_tcp_t));
+ if (!ar->stream)
+ {
+ free(ar);
+ ERR_ALLOC
+ return;
+ }
+
+ ar->tp = STREAM_API_CLIENT;
+ ar->ref = 1;
+
+ (void) uv_tcp_init(siri.loop, (uv_tcp_t *) ar->stream);
+
+ ar->stream->data = ar;
+ ar->parser.data = ar;
+
+ rc = uv_accept(server, ar->stream);
+ if (rc)
+ {
+ log_error("cannot accept HTTP API request: `%s`", uv_strerror(rc));
+ sirinet_stream_decref(ar);
+ return;
+ }
+
+ http_parser_init(&ar->parser, HTTP_REQUEST);
+
+ rc = uv_read_start(ar->stream, api__alloc_cb, api__data_cb);
+ if (rc)
+ {
+ log_error("cannot read HTTP API request: `%s`", uv_strerror(rc));
+ sirinet_stream_decref(ar);
+ return;
+ }
+}
+
+static int api__on_content_type(siri_api_request_t * ar, const char * at, size_t n)
+{
+ if (API__ICMP_WITH(at, n, api__content_type[SIRI_API_CT_JSON]) ||
+ API__ICMP_WITH(at, n, "text/json"))
+ {
+ ar->content_type = SIRI_API_CT_JSON;
+ return 0;
+ }
+
+ if (API__ICMP_WITH(at, n, api__content_type[SIRI_API_CT_QPACK]) ||
+ API__ICMP_WITH(at, n, "application/x-qpack"))
+ {
+ ar->content_type = SIRI_API_CT_QPACK;
+ return 0;
+ }
+
+ /* invalid content type */
+ log_debug("unsupported content-type: %.*s", (int) n, at);
+ return 0;
+}
+
+static int api__on_authorization(siri_api_request_t * ar, const char * at, size_t n)
+{
+ if (api__istarts_with(&at, &n, "bearer ", strlen("bearer ")))
+ {
+ log_debug("token authorization is not supported yet");
+ }
+
+ if (api__istarts_with(&at, &n, "basic ", strlen("basic ")))
+ {
+ if (ar->request_type == SIRI_APT_RT_SERVICE)
+ {
+ ar->service_authenticated = \
+ siri_service_account_check_basic(&siri, at, n);
+ return 0;
+ }
+ siridb_user_t * user;
+ user = ar->siridb
+ ? siridb_users_get_user_from_basic(ar->siridb, at, n)
+ : NULL;
+
+ if (user)
+ {
+ siridb_user_incref(user);
+ ar->origin = user;
+ }
+ return 0;
+ }
+
+ log_debug("invalid authorization type: %.*s", (int) n, at);
+ return 0;
+}
+
+static int api__header_value_cb(http_parser * parser, const char * at, size_t n)
+{
+ siri_api_request_t * ar = parser->data;
+ return ar->on_state ? ar->on_state(ar, at, n) : 0;
+}
+
+static int api__header_field_cb(http_parser * parser, const char * at, size_t n)
+{
+ siri_api_request_t * ar = parser->data;
+
+ ar->on_state = API__ICMP_WITH(at, n, "content-type")
+ ? api__on_content_type
+ : API__ICMP_WITH(at, n, "authorization")
+ ? api__on_authorization
+ : NULL;
+ return 0;
+}
+
+static int api__body_cb(http_parser * parser, const char * at, size_t n)
+{
+ size_t offset;
+ siri_api_request_t * ar = parser->data;
+
+ if (!n || !ar->len)
+ return 0;
+
+ offset = ar->len - (parser->content_length + n);
+ assert (offset + n <= ar->len);
+ memcpy(ar->buf + offset, at, n);
+
+ return 0;
+}
+
+static void api__write_cb(uv_write_t * req, int status)
+{
+ siri_api_request_t * ar = req->handle->data;
+
+ if (status)
+ log_error(
+ "error writing HTTP API response: `%s`",
+ uv_strerror(status));
+
+ /* reset the API to support multiple request on the same connection */
+ api__reset(ar);
+
+ /* Resume parsing */
+ http_parser_pause(&ar->parser, 0);
+
+ sirinet_stream_decref(ar);
+}
+
+static int api__plain_response(
+ siri_api_request_t * ar,
+ const siri_api_header_t ht)
+{
+ const char * body = api__default_body[ht];
+ char header[API__HEADER_MAX_SZ];
+ size_t body_size;
+ int header_size;
+
+ body_size = strlen(body);
+ header_size = api__header(header, ht, SIRI_API_CT_TEXT, body_size);
+
+ if (header_size > 0)
+ {
+ uv_buf_t uvbufs[2] = {
+ uv_buf_init(header, (size_t) header_size),
+ uv_buf_init((char *) body, body_size),
+ };
+
+ (void) uv_write(&ar->req, ar->stream, uvbufs, 2, api__write_cb);
+ return 0;
+ }
+ return -1;
+}
+
+static int api__query(siri_api_request_t * ar, api__query_t * q)
+{
+ sirinet_stream_incref(ar);
+
+ siridb_query_run(
+ 0,
+ (sirinet_stream_t *) ar,
+ q->query,
+ q->query_n,
+ q->factor,
+ SIRIDB_QUERY_FLAG_MASTER);
+ return 0;
+}
+
+static int api__read_factor(
+ siridb_timep_t precision,
+ qp_unpacker_t * up,
+ api__query_t * q)
+{
+ qp_obj_t obj;
+
+ if (!qp_is_raw(qp_next(up, &obj)))
+ return -1;
+
+ if (obj.len == 1 && obj.via.raw[0] == 's')
+ {
+ q->factor = pow(1000.0, SIRIDB_TIME_SECONDS - precision);
+ return 0;
+ }
+
+ if (obj.len == 2 && obj.via.raw[1] == 's')
+ {
+ switch (obj.via.raw[0])
+ {
+ case 'm':
+ q->factor = pow(1000.0, SIRIDB_TIME_MILLISECONDS - precision);
+ return 0;
+ case 'u':
+ q->factor = pow(1000.0, SIRIDB_TIME_MICROSECONDS - precision);
+ return 0;
+ case 'n':
+ q->factor = pow(1000.0, SIRIDB_TIME_NANOSECONDS - precision);
+ return 0;
+ }
+ }
+ return -1;
+}
+
+static int api__query_from_qp(api__query_t * q, siri_api_request_t * ar)
+{
+ qp_obj_t obj;
+ qp_unpacker_t up;
+ qp_unpacker_init(&up, (unsigned char *) ar->buf, ar->len);
+
+ q->factor = 0.0;
+ q->query = NULL;
+
+ if (!qp_is_map(qp_next(&up, &obj)))
+ {
+ return api__plain_response(ar, E400_BAD_REQUEST);
+ }
+
+ while (qp_next(&up, &obj) && !qp_is_close(obj.tp))
+ {
+ if (!qp_is_raw(obj.tp) || obj.len < 1)
+ {
+ return api__plain_response(ar, E400_BAD_REQUEST);
+ }
+
+ switch(*obj.via.str)
+ {
+ case 't':
+ if (api__read_factor(ar->siridb->time->precision, &up, q))
+ {
+ return api__plain_response(ar, E400_BAD_REQUEST);
+ }
+ break;
+ case 'q':
+ if (!qp_is_raw(qp_next(&up, &obj)))
+ {
+ return api__plain_response(ar, E400_BAD_REQUEST);
+ }
+ q->query = obj.via.str;
+ q->query_n = obj.len;
+ break;
+ default:
+ return api__plain_response(ar, E400_BAD_REQUEST);
+ }
+ }
+
+ return q->query == NULL
+ ? api__plain_response(ar, E400_BAD_REQUEST)
+ : api__query(ar, q);
+}
+
+static int api__insert_from_qp(siri_api_request_t * ar)
+{
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, (unsigned char *) ar->buf, ar->len);
+
+ siridb_insert_t * insert = siridb_insert_new(
+ ar->siridb,
+ 0,
+ (sirinet_stream_t *) ar);
+
+ if (insert == NULL)
+ {
+ return api__plain_response(ar, E500_INTERNAL_SERVER_ERROR);
+ }
+
+ ssize_t rc = siridb_insert_assign_pools(
+ ar->siridb,
+ &unpacker,
+ insert->packer);
+
+ switch ((siridb_insert_err_t) rc)
+ {
+ case ERR_EXPECTING_ARRAY:
+ case ERR_EXPECTING_SERIES_NAME:
+ case ERR_EXPECTING_MAP_OR_ARRAY:
+ case ERR_EXPECTING_INTEGER_TS:
+ case ERR_TIMESTAMP_OUT_OF_RANGE:
+ case ERR_UNSUPPORTED_VALUE:
+ case ERR_EXPECTING_AT_LEAST_ONE_POINT:
+ case ERR_EXPECTING_NAME_AND_POINTS:
+ case ERR_INCOMPATIBLE_SERVER_VERSION:
+ case ERR_MEM_ALLOC:
+ {
+ /* something went wrong, get correct err message */
+ const char * err_msg = siridb_insert_err_msg(rc);
+
+ log_error("Insert error: '%s' at position %lu",
+ err_msg, unpacker.pt - (unsigned char *) ar->buf);
+
+ /* create and send package */
+ sirinet_pkg_t * package = sirinet_pkg_err(
+ 0,
+ strlen(err_msg),
+ CPROTO_ERR_INSERT,
+ err_msg);
+
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send((sirinet_stream_t *) ar, package);
+ }
+ }
+
+ /* error, free insert */
+ siridb_insert_free(insert);
+ break;
+
+ default:
+ if (siridb_insert_points_to_pools(insert, (size_t) rc))
+ {
+ siridb_insert_free(insert); /* signal is raised */
+ }
+ else
+ {
+ /* extra increment for the insert task */
+ sirinet_stream_incref(ar);
+ }
+ break;
+ }
+ return 0;
+}
+
+static int api__insert_cb(http_parser * parser)
+{
+ siri_api_request_t * ar = parser->data;
+
+ if (parser->method != HTTP_POST)
+ return api__plain_response(ar, E405_METHOD_NOT_ALLOWED);
+
+ if (!ar->siridb)
+ return api__plain_response(ar, E404_NOT_FOUND);
+
+ if (!ar->origin)
+ return api__plain_response(ar, E401_UNAUTHORIZED);
+
+
+ if (!(((siridb_user_t *) ar->origin)->access_bit & SIRIDB_ACCESS_INSERT))
+ return api__plain_response(ar, E403_FORBIDDEN);
+
+ if ((
+ ar->siridb->server->flags != SERVER_FLAG_RUNNING &&
+ ar->siridb->server->flags != SERVER_FLAG_RUNNING + SERVER_FLAG_REINDEXING
+ ) ||
+ !siridb_pools_accessible(ar->siridb))
+ return api__plain_response(ar, E503_SERVICE_UNAVAILABLE);
+
+ switch (ar->content_type)
+ {
+ case SIRI_API_CT_TEXT:
+ /* Or, shall we allow text and return we some sort of CSV format? */
+ break;
+ case SIRI_API_CT_JSON:
+ {
+ char * dst;
+ size_t dst_n;
+ if (qpjson_json_to_qp(ar->buf, ar->len, &dst, &dst_n))
+ return api__plain_response(ar, E400_BAD_REQUEST);
+ free(ar->buf);
+ ar->buf = dst;
+ ar->len = dst_n;
+ }
+ /* fall through */
+ case SIRI_API_CT_QPACK:
+ return api__insert_from_qp(ar);
+ }
+
+ return api__plain_response(ar, E415_UNSUPPORTED_MEDIA_TYPE);
+}
+
+static int api__query_cb(http_parser * parser)
+{
+ api__query_t q;
+ siri_api_request_t * ar = parser->data;
+
+ if (parser->method != HTTP_POST)
+ return api__plain_response(ar, E405_METHOD_NOT_ALLOWED);
+
+ if (!ar->siridb)
+ return api__plain_response(ar, E404_NOT_FOUND);
+
+ if (!ar->origin)
+ return api__plain_response(ar, E401_UNAUTHORIZED);
+
+ switch (ar->content_type)
+ {
+ case SIRI_API_CT_TEXT:
+ /* Or, shall we allow text and return we some sort of CSV format? */
+ break;
+ case SIRI_API_CT_JSON:
+ {
+ char * dst;
+ size_t dst_n;
+ if (qpjson_json_to_qp(ar->buf, ar->len, &dst, &dst_n))
+ return api__plain_response(ar, E400_BAD_REQUEST);
+ free(ar->buf);
+ ar->buf = dst;
+ ar->len = dst_n;
+ }
+ /* fall through */
+ case SIRI_API_CT_QPACK:
+ return api__query_from_qp(&q, ar);
+ }
+
+ return api__plain_response(ar, E415_UNSUPPORTED_MEDIA_TYPE);
+}
+
+static int api__service_cb(http_parser * parser)
+{
+ qp_unpacker_t up;
+ cproto_server_t res;
+ siri_api_request_t * ar = parser->data;
+ qp_packer_t * packer = NULL;
+ sirinet_pkg_t * package;
+ char err_msg[SIRI_MAX_SIZE_ERR_MSG];
+
+ switch ((service_request_t) ar->service_type)
+ {
+ case SERVICE_NEW_ACCOUNT:
+ case SERVICE_CHANGE_PASSWORD:
+ case SERVICE_DROP_ACCOUNT:
+ case SERVICE_NEW_DATABASE:
+ case SERVICE_NEW_POOL:
+ case SERVICE_NEW_REPLICA:
+ case SERVICE_DROP_DATABASE:
+ if (parser->method != HTTP_POST)
+ return api__plain_response(ar, E405_METHOD_NOT_ALLOWED);
+ break;
+
+ case SERVICE_GET_VERSION:
+ case SERVICE_GET_ACCOUNTS:
+ case SERVICE_GET_DATABASES:
+ if (parser->method != HTTP_GET)
+ return api__plain_response(ar, E405_METHOD_NOT_ALLOWED);
+ break;
+ }
+
+ if (!ar->service_authenticated)
+ return api__plain_response(ar, E401_UNAUTHORIZED);
+
+ switch (ar->content_type)
+ {
+ case SIRI_API_CT_TEXT:
+ if (ar->len)
+ return api__plain_response(ar, E415_UNSUPPORTED_MEDIA_TYPE);
+ ar->content_type = SIRI_API_CT_JSON;
+ break;
+ case SIRI_API_CT_JSON:
+ if (ar->len)
+ {
+ char * dst;
+ size_t dst_n;
+ if (qpjson_json_to_qp(ar->buf, ar->len, &dst, &dst_n))
+ return api__plain_response(ar, E400_BAD_REQUEST);
+ free(ar->buf);
+ ar->buf = dst;
+ ar->len = dst_n;
+ }
+ break;
+ case SIRI_API_CT_QPACK:
+ break;
+ }
+
+ qp_unpacker_init(&up, (unsigned char *) ar->buf, ar->len);
+
+ res = siri_service_request(
+ ar->service_type,
+ &up,
+ &packer,
+ 0,
+ (sirinet_stream_t *) ar,
+ err_msg);
+
+ if (res == CPROTO_DEFERRED)
+ {
+ sirinet_stream_incref(ar);
+ }
+
+ package =
+ (res == CPROTO_DEFERRED) ? NULL :
+ (res == CPROTO_ERR_SERVICE) ? sirinet_pkg_err(
+ 0,
+ strlen(err_msg),
+ res,
+ err_msg) :
+ (res == CPROTO_ACK_SERVICE_DATA) ? sirinet_packer2pkg(
+ packer,
+ 0,
+ res) : sirinet_pkg_new(0, 0, res, NULL);
+
+ return package ? sirinet_pkg_send((sirinet_stream_t *) ar, package) : 0;
+}
+
+static int api__message_complete_cb(http_parser * parser)
+{
+ siri_api_request_t * ar = parser->data;
+
+ /* Pause the HTTP parser;
+ * This is required since SiriDB will handle queries and inserts
+ * asynchronously and SiriDB must be sure that the request does not
+ * change during this time. It is also important to write the responses
+ * in order and this solves both issues. */
+ http_parser_pause(&ar->parser, 1);
+
+ switch(ar->request_type)
+ {
+ case SIRI_API_RT_NONE:
+ return api__plain_response(ar, E404_NOT_FOUND);
+ case SIRI_API_RT_QUERY:
+ return api__query_cb(parser);
+ case SIRI_API_RT_INSERT:
+ return api__insert_cb(parser);
+ case SIRI_APT_RT_SERVICE:
+ return api__service_cb(parser);
+ }
+
+ return api__plain_response(ar, E500_INTERNAL_SERVER_ERROR);
+}
+
+static void api__write_free_cb(uv_write_t * req, int status)
+{
+ free(req->data);
+ api__write_cb(req, status);
+}
+
+static int api__close_resp(
+ siri_api_request_t * ar,
+ const siri_api_header_t ht,
+ void * data,
+ size_t size)
+{
+ char header[API__HEADER_MAX_SZ];
+ int header_size = 0;
+
+ header_size = api__header(header, ht, ar->content_type, size);
+
+ uv_buf_t uvbufs[2] = {
+ uv_buf_init(header, (unsigned int) header_size),
+ uv_buf_init(data, size),
+ };
+
+ /* bind response to request to we can free in the callback */
+ ar->req.data = data;
+
+ uv_write(&ar->req, ar->stream, uvbufs, 2, api__write_free_cb);
+ return 0;
+}
+
+int siri_api_init(void)
+{
+ int rc;
+ struct sockaddr_storage addr = {0};
+ uint16_t port = siri.cfg->http_api_port;
+
+ if (port == 0)
+ return 0;
+
+ if (siri.cfg->ip_support == IP_SUPPORT_IPV4ONLY)
+ {
+ (void) uv_ip4_addr("0.0.0.0", (int) port, (struct sockaddr_in *) &addr);
+ } else
+ {
+ (void) uv_ip6_addr("::", (int) port, (struct sockaddr_in6 *) &addr);
+ }
+
+ api__settings.on_url = api__url_cb;
+ api__settings.on_header_field = api__header_field_cb;
+ api__settings.on_header_value = api__header_value_cb;
+ api__settings.on_message_complete = api__message_complete_cb;
+ api__settings.on_body = api__body_cb;
+ api__settings.on_headers_complete = api__headers_complete_cb;
+
+ if (
+ (rc = uv_tcp_init(siri.loop, &api__uv_server)) ||
+ (rc = uv_tcp_bind(
+ &api__uv_server,
+ (const struct sockaddr *) &addr,
+ (siri.cfg->ip_support == IP_SUPPORT_IPV6ONLY) ? UV_TCP_IPV6ONLY : 0)) ||
+ (rc = uv_listen(
+ (uv_stream_t *) &api__uv_server,
+ 128,
+ api__connection_cb)))
+ {
+ log_error("error initializing HTTP API on port %u: `%s`",
+ port,
+ uv_strerror(rc));
+ return -1;
+ }
+
+ log_info("start listening for HTTP API requests on TCP port %u", port);
+ return 0;
+}
+
+#define API__DUP(__data, __n, __s) \
+ do { \
+ __n = strlen("{\"error_msg\":\""__s"\"}"); \
+ __data = strndup("{\"error_msg\":\""__s"\"}", __n); \
+ if (!__data) __n = 0; \
+ } while(0)
+
+static void api__yajl_parse_error(
+ char ** str,
+ size_t * size,
+ siri_api_header_t * ht,
+ yajl_gen_status stat)
+{
+ switch (stat)
+ {
+ case yajl_gen_status_ok:
+ return;
+ case yajl_gen_keys_must_be_strings:
+ *ht = E400_BAD_REQUEST;
+ API__DUP(*str, *size, "JSON keys must be strings");
+ return;
+ case yajl_max_depth_exceeded:
+ *ht = E400_BAD_REQUEST;
+ API__DUP(*str, *size, "JSON max depth exceeded");
+ return;
+ case yajl_gen_in_error_state:
+ *ht = E500_INTERNAL_SERVER_ERROR;
+ API__DUP(*str, *size, "JSON general error");
+ return;
+ case yajl_gen_generation_complete:
+ *ht = E500_INTERNAL_SERVER_ERROR;
+ API__DUP(*str, *size, "JSON completed");
+ return;
+ case yajl_gen_invalid_number:
+ *ht = E400_BAD_REQUEST;
+ API__DUP(*str, *size, "JSON invalid number");
+ return;
+ case yajl_gen_no_buf:
+ *ht = E500_INTERNAL_SERVER_ERROR;
+ API__DUP(*str, *size, "JSON no buffer has been set");
+ return;
+ case yajl_gen_invalid_string:
+ *ht = E400_BAD_REQUEST;
+ API__DUP(*str, *size, "JSON only accepts valid UTF8 strings");
+ return;
+ }
+ *ht = E500_INTERNAL_SERVER_ERROR;
+ API__DUP(*str, *size, "JSON unexpected error");
+}
+
+int siri_api_send(
+ siri_api_request_t * ar,
+ siri_api_header_t ht,
+ unsigned char * src,
+ size_t n)
+{
+ unsigned char * data;
+ if (n == 0)
+ {
+ if (ht != E200_OK)
+ {
+ return api__plain_response(ar, ht);
+ }
+ src = (unsigned char *) "\x82OK";
+ n = 3;
+ }
+
+ if (ar->content_type == SIRI_API_CT_JSON)
+ {
+ size_t tmp_sz;
+ yajl_gen_status stat = qpjson_qp_to_json(
+ src,
+ n,
+ &data,
+ &tmp_sz,
+ 0);
+ if (stat)
+ api__yajl_parse_error((char **) &data, &tmp_sz, &ht, stat);
+
+ n = tmp_sz;
+ }
+ else
+ {
+ assert (ar->content_type == SIRI_API_CT_QPACK);
+ data = malloc(n);
+ if (data == NULL)
+ {
+ ht = E500_INTERNAL_SERVER_ERROR;
+ n = 0;
+ }
+ else
+ {
+ memcpy(data, src, n);
+ }
+ }
+
+ return api__close_resp(ar, ht, data, n);
+}
--- /dev/null
+/*
+ * args.c - Parse SiriDB command line arguments.
+ */
+#include <siri/args/args.h>
+#include <argparse/argparse.h>
+#include <siri/version.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+
+#define DEFAULT_LOG_FILE_MAX_SIZE 50000000
+#define DEFAULT_LOG_FILE_NUM_BACKUPS 6
+
+#ifndef NDEBUG
+#define DEFAULT_LOG_LEVEL "debug"
+#else
+#define DEFAULT_LOG_LEVEL "warning"
+#endif
+
+static siri_args_t siri_args = {
+ .version=0,
+ .config="",
+ .log_level="",
+ .log_colorized=0,
+ .managed=0,
+};
+
+void siri_args_parse(siri_t * siri, int argc, char *argv[])
+{
+ siri->args = &siri_args;
+
+ argparse_parser_t parser;
+ argparse_init(&parser);
+
+ argparse_argument_t config = {
+ "config", /* name */
+ 'c', /* shortcut */
+ "define which SiriDB configuration file " /* help */
+ "to use",
+ ARGPARSE_STORE_STRING, /* action */
+ 0, /* default int32_t */
+ NULL, /* value pt_int32_t */
+ "/etc/siridb/siridb.conf", /* default string */
+ siri_args.config, /* value string */
+ NULL /* choices */
+ };
+
+ argparse_argument_t version = {
+ "version", /* name */
+ 'v', /* shortcut */
+ "show version information and exit", /* help */
+ ARGPARSE_STORE_TRUE, /* action */
+ 0, /* default int32_t */
+ &siri_args.version, /* value pt_int32_t */
+ NULL, /* default string */
+ NULL, /* value string */
+ NULL /* choices */
+ };
+
+ argparse_argument_t managed = {
+ "managed",
+ 0,
+ "use this flag when deployed using a managed environment",
+ ARGPARSE_STORE_TRUE,
+ 0,
+ &siri_args.managed,
+ NULL,
+ NULL,
+ NULL,
+ };
+
+ argparse_argument_t log_level = {
+ "log-level", /* name */
+ 'l', /* shortcut */
+ "set initial log level", /* help */
+ ARGPARSE_STORE_STR_CHOICE, /* action */
+ 0, /* default int32_t */
+ NULL, /* value pt_int32_t */
+ DEFAULT_LOG_LEVEL, /* default string */
+ siri_args.log_level, /* value string */
+ "debug,info,warning,error,critical" /* choices */
+ };
+
+ argparse_argument_t log_colorized = {
+ "log-colorized", /* name */
+ 0, /* shortcut */
+ "use colorized logging", /* help */
+ ARGPARSE_STORE_TRUE, /* action */
+ 0, /* default int32_t */
+ &siri_args.log_colorized, /* value pt_int32_t */
+ NULL, /* default string */
+ NULL, /* value string */
+ NULL /* choices */
+ };
+
+ argparse_add_argument(&parser, &config);
+ argparse_add_argument(&parser, &version);
+ argparse_add_argument(&parser, &log_level);
+ argparse_add_argument(&parser, &log_colorized);
+ argparse_add_argument(&parser, &managed);
+
+ /* this will parse and free the parser from memory */
+ argparse_parse(&parser, argc, argv);
+
+ if (siri_args.version)
+ {
+ printf(
+ "SiriDB Server %s\n"
+ "Contributers: %s\n"
+ "Home-page: %s\n",
+ SIRIDB_VERSION,
+ SIRIDB_CONTRIBUTERS,
+ SIRIDB_HOME_PAGE);
+
+ exit(EXIT_SUCCESS);
+ }
+}
--- /dev/null
+/*
+ * async.c - SiriDB async wrapper.
+ */
+#include <logger/logger.h>
+#include <siri/async.h>
+
+/*
+ * Used as uv_close_cb for closing uv_async_t
+ */
+void siri_async_close(uv_handle_t * handle)
+{
+ siri_async_decref((uv_async_t **) &handle);
+}
+
+/*
+ * Decrement reference counter for handle. If zero is reached the handle
+ * will be freed and handle will be set to NULL.
+ */
+void siri_async_decref(uv_async_t ** handle)
+{
+ siri_async_t * shandle = (*handle)->data;
+ if (!--shandle->ref)
+ {
+ shandle->free_cb((uv_handle_t *) *handle);
+ *handle = NULL;
+ }
+}
+
+
--- /dev/null
+/*
+ * backup.c - Set SiriDB in backup mode.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/backup.h>
+#include <siri/db/replicate.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/db/shard.h>
+#include <siri/db/shards.h>
+#include <siri/optimize.h>
+#include <siri/siri.h>
+#include <stddef.h>
+
+uv_timer_t backup;
+
+#define BACKUP_LOOP_TIMEOUT 3000
+
+static void BACKUP_cb(uv_timer_t * timer);
+static int BACKUP_walk(siridb_t * siridb, void * args);
+
+int siri_backup_init(siri_t * siri)
+{
+ siri->backup = &backup;
+ siri->backup->data = llist_new();
+ if (siri->backup->data == NULL)
+ {
+ return -1;
+ }
+
+ uv_timer_init(siri->loop, siri->backup);
+ return 0;
+}
+
+void siri_backup_destroy(siri_t * siri)
+{
+ llist_t * llist = (llist_t *) siri->backup->data;
+
+ if (llist != NULL)
+ {
+ if (llist->len)
+ {
+ uv_timer_stop(siri->backup);
+ }
+
+ llist_free_cb(llist, (llist_cb) siridb_decref_cb, NULL);
+ }
+
+ uv_close((uv_handle_t *) siri->backup, NULL);
+}
+
+/*
+ * Returns 0 when successful or -1 and a signal might be raised in case
+ * of an error.
+ */
+int siri_backup_enable(siri_t * siri, siridb_t * siridb)
+{
+ assert (~siridb->server->flags & SERVER_FLAG_BACKUP_MODE);
+ assert (~siridb->server->flags & SERVER_FLAG_REINDEXING);
+
+ llist_t * llist = (llist_t *) siri->backup->data;
+
+ if (llist_append(llist, siridb))
+ {
+ return -1;
+ }
+
+ siridb_incref(siridb);
+
+ siridb->server->flags |= SERVER_FLAG_BACKUP_MODE;
+
+ siridb_servers_send_flags(siridb->servers);
+
+ if (~siridb->server->flags & SERVER_FLAG_SYNCHRONIZING)
+ {
+ siri_optimize_pause();
+ }
+
+ if (siridb->replicate != NULL)
+ {
+ siridb_replicate_pause(siridb->replicate);
+ }
+
+ if (llist->len == 1)
+ {
+ uv_timer_start(
+ siri->backup,
+ BACKUP_cb,
+ BACKUP_LOOP_TIMEOUT,
+ BACKUP_LOOP_TIMEOUT);
+ }
+
+ return 0;
+}
+
+int siri_backup_disable(siri_t * siri, siridb_t * siridb)
+{
+ assert (siridb->server->flags & SERVER_FLAG_BACKUP_MODE);
+
+ int rc = 0;
+
+ llist_t * llist = (llist_t *) siri->backup->data;
+
+ if ((siridb_t *) llist_remove(llist, NULL, siridb) == siridb)
+ {
+ siridb_decref(siridb);
+ }
+ else
+ {
+ log_critical("Cannot find SiriDB in backup list");
+ rc = -1;
+ }
+
+ siridb->server->flags &= ~SERVER_FLAG_BACKUP_MODE;
+
+ if (siridb->fifo != NULL)
+ {
+ if (siridb->fifo->in->fp == NULL && siridb_fifo_open(siridb->fifo))
+ {
+ log_critical("Cannot open fifo file");
+ rc = -1;
+ }
+ }
+
+ if (siridb->replicate == NULL)
+ {
+ /*
+ * Optimize will be continued when synchronization has finished or now
+ * if we do not have a replica server.
+ */
+ siri_optimize_continue();
+ }
+ else
+ {
+ siridb->server->flags |= SERVER_FLAG_SYNCHRONIZING;
+ siridb_replicate_continue(siridb->replicate);
+ }
+
+ if (llist->len == 0)
+ {
+ uv_timer_stop(siri->backup);
+ }
+
+ siridb_servers_send_flags(siridb->servers);
+
+ return rc;
+}
+
+static void BACKUP_cb(uv_timer_t * timer)
+{
+ llist_walk((llist_t *) timer->data, (llist_cb) BACKUP_walk, NULL);
+}
+
+static int BACKUP_walk(siridb_t * siridb, void * args __attribute__((unused)))
+{
+ if ( siridb->replicate != NULL &&
+ siridb->replicate->status == REPLICATE_PAUSED &&
+ siridb->fifo != NULL &&
+ siridb->fifo->in->fp != NULL)
+ {
+ siridb_fifo_close(siridb->fifo);
+ }
+
+ if (siridb->buffer->fp != NULL)
+ {
+ if (fclose(siridb->buffer->fp) == 0)
+ {
+ siridb->buffer->fp = NULL;
+ }
+ else
+ {
+ log_critical("Cannot close buffer file");
+ }
+ }
+
+ if (siridb->dropped_fp != NULL)
+ {
+ if (fclose(siridb->dropped_fp) == 0)
+ {
+ siridb->dropped_fp = NULL;
+ }
+ else
+ {
+ log_critical("Cannot close dropped file");
+ }
+ }
+
+ if (siridb->store != NULL)
+ {
+ if (qp_close(siridb->store) == 0)
+ {
+ siridb->store = NULL;
+ }
+ else
+ {
+ log_critical("Cannot close store file");
+ }
+ }
+
+ if (SIRI_OPTIMZE_IS_PAUSED)
+ {
+ size_t i;
+ siridb_shard_t * shard;
+
+ /*
+ * A lock is not needed since the optimize thread is paused and this
+ * is running from the main thread.
+ */
+ vec_t * shard_list = siridb_shards_vec(siridb);
+
+ if (shard_list == NULL)
+ {
+ log_critical(
+ "Cannot create shard list so not all shard files "
+ "will be closed in backup mode.");
+ }
+ else for (i = 0; i < shard_list->len; i++)
+ {
+ shard = (siridb_shard_t *) shard_list->data[i];
+
+ if (shard->fp->fp != NULL)
+ {
+ siri_fp_close(shard->fp);
+ }
+
+ if (shard->replacing != NULL && shard->replacing->fp->fp != NULL)
+ {
+ siri_fp_close(shard->replacing->fp);
+ }
+ siridb_shard_decref(shard);
+ }
+
+ vec_free(shard_list);
+ }
+
+ return 0;
+}
--- /dev/null
+/*
+ * buffersync.c - Buffer sync.
+ */
+#include <logger/logger.h>
+#include <siri/db/server.h>
+#include <siri/db/buffer.h>
+#include <siri/buffersync.h>
+#include <uv.h>
+
+
+static uv_timer_t buffersync;
+
+#define BUFFERSYNC_INIT_TIMEOUT 1000
+
+static void BUFFERSYNC_cb(uv_timer_t * handle);
+
+void siri_buffersync_init(siri_t * siri)
+{
+ uint64_t repeat = (uint64_t) siri->cfg->buffer_sync_interval;
+ if (repeat == 0)
+ {
+ siri->buffersync = NULL;
+ return;
+ }
+ siri->buffersync = &buffersync;
+ uv_timer_init(siri->loop, &buffersync);
+ uv_timer_start(
+ &buffersync,
+ BUFFERSYNC_cb,
+ BUFFERSYNC_INIT_TIMEOUT,
+ repeat < 100 ? 100 : repeat);
+}
+
+void siri_buffersync_stop(siri_t * siri)
+{
+ if (siri->buffersync != NULL)
+ {
+ /* stop the timer so it will not run again */
+ uv_timer_stop(&buffersync);
+ uv_close((uv_handle_t *) &buffersync, NULL);
+ siri->buffersync = NULL;
+ }
+}
+
+
+static void BUFFERSYNC_cb(uv_timer_t * handle __attribute__((unused)))
+{
+ siridb_t * siridb;
+ llist_node_t * siridb_node;
+
+ siridb_node = siri.siridb_list->first;
+
+ while (siridb_node != NULL)
+ {
+ siridb = (siridb_t *) siridb_node->data;
+
+ /* flush the buffer, maybe on each insert or another interval? */
+ if (siridb_buffer_fsync(siridb->buffer))
+ {
+ log_critical("fsync() has failed on the buffer file");
+ }
+
+ siridb_node = siridb_node->next;
+ }
+}
+
--- /dev/null
+/*
+ * cfg.h - Read the global SiriDB configuration file. (usually siridb.conf)
+ */
+#include <cfgparser/cfgparser.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <logger/logger.h>
+#include <siri/cfg/cfg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <xstr/xstr.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/resource.h>
+#include <siri/net/tcp.h>
+
+static siri_cfg_t siri_cfg = {
+ .http_status_port=0, /* 0=disabled, 1-16535=enabled */
+ .http_api_port=0, /* 0=disabled, 1-16535=enabled */
+ .listen_client_port=9000,
+ .listen_backend_port=9010,
+ .bind_client_addr=NULL,
+ .bind_backend_addr=NULL,
+ .heartbeat_interval=30,
+ .max_open_files=DEFAULT_OPEN_FILES_LIMIT,
+ .optimize_interval=3600,
+ .ip_support=IP_SUPPORT_ALL,
+ .shard_compression=0,
+ .shard_auto_duration=0,
+ .server_address="localhost",
+ .db_path="",
+ .pipe_support=0,
+ .pipe_client_name="siridb_client.sock",
+ .buffer_sync_interval=0,
+};
+
+static void SIRI_CFG_read_uint(
+ cfgparser_t * cfgparser,
+ const char * option_name,
+ int min,
+ int max,
+ uint32_t * value);
+static void SIRI_CFG_read_address_port(
+ cfgparser_t * cfgparser,
+ const char * option_name,
+ char * address_pt,
+ uint16_t * port_pt);
+static void SIRI_CFG_read_addr(
+ cfgparser_t * cfgparser,
+ const char * option_name,
+ char ** dest);
+static void SIRI_CFG_read_pipe_client_name(cfgparser_t * cfgparser);
+static void SIRI_CFG_read_db_path(cfgparser_t * cfgparser);
+static void SIRI_CFG_read_max_open_files(cfgparser_t * cfgparser);
+static void SIRI_CFG_read_ip_support(cfgparser_t * cfgparser);
+static void SIRI_CFG_read_shard_compression(cfgparser_t * cfgparser);
+static void SIRI_CFG_read_shard_auto_duration(cfgparser_t * cfgparser);
+static void SIRI_CFG_read_pipe_support(cfgparser_t * cfgparser);
+
+void siri_cfg_init(siri_t * siri)
+{
+ /* Read the application configuration file. */
+ cfgparser_t * cfgparser = cfgparser_new();
+ cfgparser_return_t rc;
+ uint32_t tmp;
+ siri->cfg = &siri_cfg;
+ rc = cfgparser_read(cfgparser, siri->args->config);
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ /* we could choose to continue with defaults but this is probably
+ * not what users want so lets quit.
+ */
+ log_debug(
+ "Not using configuration file '%s' (%s)",
+ siri->args->config,
+ cfgparser_errmsg(rc));
+ cfgparser_free(cfgparser);
+ return;
+ }
+
+ SIRI_CFG_read_address_port(
+ cfgparser,
+ "server_name",
+ siri_cfg.server_address,
+ &siri_cfg.listen_backend_port);
+
+ tmp = siri_cfg.listen_client_port;
+
+ SIRI_CFG_read_uint(
+ cfgparser,
+ "listen_client_port",
+ 1,
+ 65535,
+ &tmp);
+ siri_cfg.listen_client_port = (uint16_t) tmp;
+
+ SIRI_CFG_read_uint(
+ cfgparser,
+ "optimize_interval",
+ 0,
+ 2419200, /* 4 weeks */
+ &siri_cfg.optimize_interval);
+
+ tmp = siri_cfg.heartbeat_interval;
+ SIRI_CFG_read_uint(
+ cfgparser,
+ "heartbeat_interval",
+ 3,
+ 300,
+ &tmp);
+ siri_cfg.heartbeat_interval = (uint16_t) tmp;
+
+ tmp = siri_cfg.http_status_port;
+ SIRI_CFG_read_uint(
+ cfgparser,
+ "http_status_port",
+ 0,
+ 65535,
+ &tmp);
+ siri_cfg.http_status_port = (uint16_t) tmp;
+
+ tmp = siri_cfg.http_api_port;
+ SIRI_CFG_read_uint(
+ cfgparser,
+ "http_api_port",
+ 0,
+ 65535,
+ &tmp);
+ siri_cfg.http_api_port = (uint16_t) tmp;
+
+ SIRI_CFG_read_db_path(cfgparser);
+ SIRI_CFG_read_max_open_files(cfgparser);
+ SIRI_CFG_read_ip_support(cfgparser);
+ SIRI_CFG_read_shard_compression(cfgparser);
+ SIRI_CFG_read_shard_auto_duration(cfgparser);
+
+ SIRI_CFG_read_addr(
+ cfgparser,
+ "bind_client_address",
+ &siri_cfg.bind_client_addr);
+
+ SIRI_CFG_read_addr(
+ cfgparser,
+ "bind_server_address",
+ &siri_cfg.bind_backend_addr);
+
+ SIRI_CFG_read_pipe_support(cfgparser);
+
+ if (siri_cfg.pipe_support)
+ {
+ SIRI_CFG_read_pipe_client_name(cfgparser);
+ }
+
+ tmp = siri_cfg.buffer_sync_interval;
+ SIRI_CFG_read_uint(
+ cfgparser,
+ "buffer_sync_interval",
+ 0,
+ 300000,
+ &tmp);
+ siri_cfg.buffer_sync_interval = (uint32_t) tmp;
+
+ cfgparser_free(cfgparser);
+}
+
+void siri_cfg_destroy(siri_t * siri)
+{
+ free(siri->cfg->bind_backend_addr);
+ free(siri->cfg->bind_client_addr);
+}
+
+static void SIRI_CFG_read_uint(
+ cfgparser_t * cfgparser,
+ const char * option_name,
+ int min,
+ int max,
+ uint32_t * value)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ option_name);
+
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ log_warning(
+ "Missing '%s' in '%s' (%s). ",
+ option_name,
+ siri.args->config,
+ cfgparser_errmsg(rc));
+ return;
+ }
+
+ if (option->tp != CFGPARSER_TP_INTEGER)
+ {
+ log_warning(
+ "Error reading '%s' in '%s': %s.",
+ option_name,
+ siri.args->config,
+ "error: expecting an integer value");
+ return;
+ }
+
+ if (option->val->integer < min || option->val->integer > max)
+ {
+ log_warning(
+ "Error reading '%s' in '%s': "
+ "error: value should be between %d and %d but got %d.",
+ option_name,
+ siri.args->config,
+ min,
+ max,
+ option->val->integer);
+ return;
+ }
+ *value = option->val->integer;
+}
+
+static void SIRI_CFG_read_ip_support(cfgparser_t * cfgparser)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ "ip_support");
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ log_warning(
+ "Missing 'ip_support' in '%s' (%s).",
+ siri.args->config,
+ cfgparser_errmsg(rc));
+ }
+ else if (option->tp != CFGPARSER_TP_STRING)
+ {
+ log_warning(
+ "Error reading 'ip_support' in '%s': %s.",
+ siri.args->config,
+ "error: expecting a string value");
+ }
+ else
+ {
+ if (strcmp(option->val->string, "ALL") == 0)
+ {
+ siri_cfg.ip_support = IP_SUPPORT_ALL;
+ }
+ else if (strcmp(option->val->string, "IPV4ONLY") == 0)
+ {
+ siri_cfg.ip_support = IP_SUPPORT_IPV4ONLY;
+ }
+ else if (strcmp(option->val->string, "IPV6ONLY") == 0)
+ {
+ siri_cfg.ip_support = IP_SUPPORT_IPV6ONLY;
+ }
+ else
+ {
+ log_warning(
+ "Error reading 'ip_support' in '%s': "
+ "error: expecting ALL, IPV4ONLY or IPV6ONLY but got '%s'.",
+ siri.args->config,
+ option->val->string);
+ }
+ }
+}
+
+static void SIRI_CFG_read_shard_compression(cfgparser_t * cfgparser)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ "enable_shard_compression");
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ log_warning(
+ "Missing 'enable_shard_compression' in '%s' (%s).",
+ siri.args->config,
+ cfgparser_errmsg(rc));
+ }
+ else if (option->tp != CFGPARSER_TP_INTEGER || option->val->integer > 1)
+ {
+ log_warning(
+ "Error reading 'enable_shard_compression' in '%s': %s.",
+ siri.args->config,
+ "error: expecting 0 or 1");
+ }
+ else if (option->val->integer == 1)
+ {
+ siri_cfg.shard_compression = 1;
+ }
+}
+
+static void SIRI_CFG_read_shard_auto_duration(cfgparser_t * cfgparser)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ "enable_shard_auto_duration");
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ log_warning(
+ "Missing 'enable_shard_auto_duration' in '%s' (%s).",
+ siri.args->config,
+ cfgparser_errmsg(rc));
+ }
+ else if (option->tp != CFGPARSER_TP_INTEGER || option->val->integer > 1)
+ {
+ log_warning(
+ "Error reading 'enable_shard_auto_duration' in '%s': %s.",
+ siri.args->config,
+ "error: expecting 0 or 1");
+ }
+ else if (option->val->integer == 1)
+ {
+ siri_cfg.shard_auto_duration = 1;
+ }
+}
+
+static void SIRI_CFG_read_pipe_support(cfgparser_t * cfgparser)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ "enable_pipe_support");
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ log_warning(
+ "Missing 'enable_pipe_support' in '%s' (%s).",
+ siri.args->config,
+ cfgparser_errmsg(rc));
+ }
+ else if (option->tp != CFGPARSER_TP_INTEGER || option->val->integer > 1)
+ {
+ log_warning(
+ "Error reading 'enable_pipe_support' in '%s': %s.",
+ siri.args->config,
+ "error: expecting 0 or 1");
+ }
+ else if (option->val->integer == 1)
+ {
+ siri_cfg.pipe_support = 1;
+ }
+}
+
+static void SIRI_CFG_read_addr(
+ cfgparser_t * cfgparser,
+ const char * option_name,
+ char ** dest)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+ struct in_addr sa;
+ struct in6_addr sa6;
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ option_name);
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ return;
+ }
+ if (option->tp != CFGPARSER_TP_STRING)
+ {
+ log_error(
+ "Error reading '%s' in '%s': %s. ",
+ option_name,
+ siri.args->config,
+ "error: expecting a string value");
+ return;
+ }
+ if (!inet_pton(AF_INET, option->val->string, &sa) &&
+ !inet_pton(AF_INET6, option->val->string, &sa6))
+ {
+ log_error(
+ "Error reading '%s' in '%s': %s. ",
+ option_name,
+ siri.args->config,
+ "error: expecting a valid IPv4 or IPv6 address");
+ return;
+ }
+ *dest = strdup(option->val->string);
+ if (!(*dest))
+ {
+ log_error("Error allocating memory for address.");
+ }
+}
+
+static void SIRI_CFG_read_pipe_client_name(cfgparser_t * cfgparser)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+ size_t len;
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ "pipe_client_name");
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ log_warning(
+ "Missing 'pipe_client_name' in '%s' (%s).",
+ siri.args->config,
+ cfgparser_errmsg(rc));
+ return;
+ }
+
+ if (option->tp != CFGPARSER_TP_STRING)
+ {
+ log_warning(
+ "Error reading 'pipe_client_name' in '%s': %s.",
+ siri.args->config,
+ "error: expecting a string value");
+ return;
+ }
+
+ len = strlen(option->val->string);
+ if (len > XPATH_MAX-2)
+ {
+ log_warning(
+ "Pipe client name exceeds %d characters, please "
+ "check your configuration file: %s.",
+ XPATH_MAX-2,
+ siri.args->config);
+ return;
+ }
+
+ if (len == 0)
+ {
+ log_warning(
+ "Pipe client should not be an empty string, please "
+ "check your configuration file: %s.",
+ siri.args->config);
+ return;
+ }
+
+ strcpy(siri_cfg.pipe_client_name, option->val->string);
+}
+
+static void SIRI_CFG_read_db_path(cfgparser_t * cfgparser)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ "db_path");
+
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ /* Fall-back using the old configuration name */
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ "default_db_path");
+
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ return;
+ }
+ }
+
+ if (option->tp != CFGPARSER_TP_STRING)
+ {
+ log_warning(
+ "Error reading 'db_path' in '%s': %s.",
+ siri.args->config,
+ "error: expecting a string value");
+ return;
+ }
+
+ if (strlen(option->val->string) >= XPATH_MAX)
+ {
+ log_warning(
+ "Error reading 'db_path' in '%s': %s.",
+ siri.args->config,
+ "error: path too long");
+ return;
+ }
+
+ strncpy(siri_cfg.db_path, option->val->string, XPATH_MAX);
+}
+
+static void SIRI_CFG_read_max_open_files(cfgparser_t * cfgparser)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ "max_open_files");
+ if (rc != CFGPARSER_SUCCESS || option->tp != CFGPARSER_TP_INTEGER)
+ {
+ return;
+ }
+
+ siri_cfg.max_open_files = (uint16_t) option->val->integer;
+}
+
+/*
+ * Note that address_pt must have a size of at least SIRI_CFG_MAX_LEN_ADDRESS.
+ */
+static void SIRI_CFG_read_address_port(
+ cfgparser_t * cfgparser,
+ const char * option_name,
+ char * address_pt,
+ uint16_t * port_pt)
+{
+ cfgparser_option_t * option;
+ cfgparser_return_t rc;
+
+ rc = cfgparser_get_option(
+ &option,
+ cfgparser,
+ "siridb",
+ option_name);
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ return;
+ }
+
+ if (option->tp != CFGPARSER_TP_STRING)
+ {
+ log_critical(
+ "Error reading '%s' in '%s': %s. ",
+ option_name,
+ siri.args->config,
+ "error: expecting a string value");
+ exit(EXIT_FAILURE);
+ return;
+ }
+
+ if (sirinet_extract_addr_port(option->val->string, address_pt, port_pt))
+ {
+ exit(EXIT_FAILURE);
+ return;
+ }
+}
--- /dev/null
+/*
+ * access.c - Access constants and functions.
+ */
+#include <siri/db/access.h>
+#include <stdio.h>
+#include <string.h>
+
+#define ACCESS_SIZE 14
+
+static const siridb_access_repr_t access_map[ACCESS_SIZE] = {
+
+ /* access profiles, biggest numbers must be fist */
+ {.repr="full", .access_bit=SIRIDB_ACCESS_PROFILE_FULL},
+ {.repr="modify", .access_bit=SIRIDB_ACCESS_PROFILE_MODIFY},
+ {.repr="write", .access_bit=SIRIDB_ACCESS_PROFILE_WRITE},
+ {.repr="read", .access_bit=SIRIDB_ACCESS_PROFILE_READ},
+
+ /* access bits, order here is not important */
+ {.repr="alter", .access_bit=SIRIDB_ACCESS_ALTER},
+ {.repr="count", .access_bit=SIRIDB_ACCESS_COUNT},
+ {.repr="create", .access_bit=SIRIDB_ACCESS_CREATE},
+ {.repr="drop", .access_bit=SIRIDB_ACCESS_DROP},
+ {.repr="grant", .access_bit=SIRIDB_ACCESS_GRANT},
+ {.repr="insert", .access_bit=SIRIDB_ACCESS_INSERT},
+ {.repr="list", .access_bit=SIRIDB_ACCESS_LIST},
+ {.repr="revoke", .access_bit=SIRIDB_ACCESS_REVOKE},
+ {.repr="select", .access_bit=SIRIDB_ACCESS_SELECT},
+ {.repr="show", .access_bit=SIRIDB_ACCESS_SHOW},
+};
+
+/*
+ * Returns access bit by string.
+ */
+uint32_t siridb_access_from_strn(const char * str, size_t n)
+{
+ int i;
+ for (i = 0; i < ACCESS_SIZE; i++)
+ {
+ if (strncmp(access_map[i].repr, str, n) == 0)
+ {
+ return access_map[i].access_bit;
+ }
+ }
+ return 0;
+}
+
+/*
+ * Returns a access bit flag from a Cleri children object.
+ */
+uint32_t siridb_access_from_children(cleri_children_t * children)
+{
+ uint32_t access_bit = 0;
+
+ while (children != NULL)
+ {
+ access_bit |= siridb_access_from_strn(
+ children->node->str,
+ children->node->len);
+ if (children->next == NULL)
+ break;
+ children = children->next->next;
+ }
+
+ return access_bit;
+}
+
+/*
+ * Make sure 'str' is a pointer to a string which can hold at least
+ * SIRIDB_ACCESS_STR_MAX.
+ */
+void siridb_access_to_str(char * str, uint32_t access_bit)
+{
+ char * pt = str;
+ int i;
+
+ for (i = 0; i < ACCESS_SIZE && access_bit; i++)
+ {
+ if ((access_bit & access_map[i].access_bit) == access_map[i].access_bit)
+ {
+ access_bit -= access_map[i].access_bit;
+ pt += (pt == str) ? sprintf(pt, "%s", access_map[i].repr) :
+ (access_bit) ? sprintf(pt, ", %s", access_map[i].repr) :
+ sprintf(pt, " and %s", access_map[i].repr);
+ }
+ }
+ if (pt == str)
+ {
+ sprintf(pt, "no access");
+ }
+}
--- /dev/null
+/*
+ * aggregate.c - SiriDB aggregation methods.
+ */
+#include <assert.h>
+#include <limits.h>
+#include <logger/logger.h>
+#include <siri/db/aggregate.h>
+#include <siri/db/median.h>
+#include <siri/db/variance.h>
+#include <siri/grammar/grammar.h>
+#include <siri/db/re.h>
+#include <vec/vec.h>
+#include <stddef.h>
+#include <xstr/xstr.h>
+#include <math.h>
+
+#define AGGR_NEW \
+if ((aggr = AGGREGATE_new(gid)) == NULL) \
+{ \
+ sprintf(err_msg, "Memory allocation error."); \
+ siridb_aggregate_list_free(vec); \
+ return NULL; \
+}
+
+#define VEC_APPEND \
+if (vec_append_safe(&vec, aggr)) \
+{ \
+ sprintf(err_msg, "Memory allocation error."); \
+ AGGREGATE_free(aggr); \
+ siridb_aggregate_list_free(vec); \
+ return NULL; \
+}
+
+typedef int (* AGGR_cb)(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+#define GROUP_TS(point) \
+ (point->ts + aggr->group_by - 1) / aggr->group_by * aggr->group_by + \
+ aggr->offset
+
+static AGGR_cb AGGREGATES[F_OFFSET];
+
+static siridb_aggr_t * AGGREGATE_new(uint32_t gid);
+static int AGGREGATE_regex_cmp(siridb_aggr_t * aggr, char * val);
+static void AGGREGATE_free(siridb_aggr_t * aggr);
+static int AGGREGATE_init_filter(
+ siridb_aggr_t * aggr,
+ cleri_node_t * node,
+ char * err_msg);
+static siridb_points_t * AGGREGATE_limit(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+static siridb_points_t * AGGREGATE_derivative(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+static siridb_points_t * AGGREGATE_difference(
+ siridb_points_t * source,
+ char * err_msg);
+static siridb_points_t * AGGREGATE_timeval(
+ siridb_points_t * source,
+ char * err_msg);
+static siridb_points_t * AGGREGATE_interval(
+ siridb_points_t * source,
+ char * err_msg);
+static siridb_points_t * AGGREGATE_filter(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+static siridb_points_t * AGGREGATE_to_one(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+static siridb_points_t * AGGREGATE_group_by(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_count(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_derivative(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_difference(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_max(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_mean(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_median(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_median_high(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_median_low(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_min(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_pvariance(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_sum(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_variance(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_stddev(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_first(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+static int aggr_last(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg);
+
+/*
+ * Initialize aggregates.
+ */
+void siridb_init_aggregates(void)
+{
+ uint_fast16_t i;
+ for (i = 0; i < F_OFFSET; i++)
+ {
+ AGGREGATES[i] = NULL;
+ }
+
+ /* filter is not included since we only use these for group_by functions */
+ AGGREGATES[CLERI_GID_F_COUNT - F_OFFSET] = aggr_count;
+ AGGREGATES[CLERI_GID_F_DERIVATIVE - F_OFFSET] = aggr_derivative;
+ AGGREGATES[CLERI_GID_F_DIFFERENCE - F_OFFSET] = aggr_difference;
+ AGGREGATES[CLERI_GID_F_MAX - F_OFFSET] = aggr_max;
+ AGGREGATES[CLERI_GID_F_MEAN - F_OFFSET] = aggr_mean;
+ AGGREGATES[CLERI_GID_F_MEDIAN - F_OFFSET] = aggr_median;
+ AGGREGATES[CLERI_GID_F_MEDIAN_HIGH - F_OFFSET] = aggr_median_high;
+ AGGREGATES[CLERI_GID_F_MEDIAN_LOW - F_OFFSET] = aggr_median_low;
+ AGGREGATES[CLERI_GID_F_MIN - F_OFFSET] = aggr_min;
+ AGGREGATES[CLERI_GID_F_PVARIANCE - F_OFFSET] = aggr_pvariance;
+ AGGREGATES[CLERI_GID_F_SUM - F_OFFSET] = aggr_sum;
+ AGGREGATES[CLERI_GID_F_VARIANCE - F_OFFSET] = aggr_variance;
+ AGGREGATES[CLERI_GID_F_STDDEV - F_OFFSET] = aggr_stddev;
+ AGGREGATES[CLERI_GID_F_FIRST - F_OFFSET] = aggr_first;
+ AGGREGATES[CLERI_GID_F_LAST - F_OFFSET] = aggr_last;
+}
+
+/*
+ * Returns NULL in case an error has occurred and the err_msg will be set.
+ */
+vec_t * siridb_aggregate_list(cleri_children_t * children, char * err_msg)
+{
+ uint32_t gid;
+ siridb_aggr_t * aggr;
+ vec_t * vec = vec_new(VEC_DEFAULT_SIZE);
+ if (vec == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ return NULL;
+ }
+
+ /* Loop over all aggregations */
+ while (1)
+ {
+ gid = children->node->children->node->cl_obj->gid;
+
+ switch (gid)
+ {
+ case CLERI_GID_F_LIMIT:
+ AGGR_NEW
+ {
+ int64_t limit = CLERI_NODE_DATA(children->node->children->node->
+ children->next->next->node);
+
+ if (limit <= 0)
+ {
+ sprintf(err_msg,
+ "Limit must be an integer value "
+ "larger than zero.");
+ AGGREGATE_free(aggr);
+ siridb_aggregate_list_free(vec);
+ return NULL;
+ }
+
+ aggr->limit = limit;
+
+ gid = children->node->children->node->children->next->
+ next->next->next->node->children->node->
+ cl_obj->gid;
+
+ switch (gid)
+ {
+ case CLERI_GID_K_MEAN:
+ aggr->gid = CLERI_GID_F_MEAN;
+ break;
+
+ case CLERI_GID_K_MEDIAN:
+ aggr->gid = CLERI_GID_F_MEDIAN;
+ break;
+
+ case CLERI_GID_K_MEDIAN_LOW:
+ aggr->gid = CLERI_GID_F_MEDIAN_LOW;
+ break;
+
+ case CLERI_GID_K_MEDIAN_HIGH:
+ aggr->gid = CLERI_GID_F_MEDIAN_HIGH;
+ break;
+
+ case CLERI_GID_K_SUM:
+ aggr->gid = CLERI_GID_F_SUM;
+ break;
+
+ case CLERI_GID_K_MIN:
+ aggr->gid = CLERI_GID_F_MIN;
+ break;
+
+ case CLERI_GID_K_MAX:
+ aggr->gid = CLERI_GID_F_MAX;
+ break;
+
+ case CLERI_GID_K_COUNT:
+ aggr->gid = CLERI_GID_F_COUNT;
+ break;
+
+ case CLERI_GID_K_VARIANCE:
+ aggr->gid = CLERI_GID_F_VARIANCE;
+ break;
+
+ case CLERI_GID_K_PVARIANCE:
+ aggr->gid = CLERI_GID_F_PVARIANCE;
+ break;
+
+ case CLERI_GID_K_STDDEV:
+ aggr->gid = CLERI_GID_F_STDDEV;
+ break;
+
+ case CLERI_GID_K_FIRST:
+ aggr->gid = CLERI_GID_F_FIRST;
+ break;
+
+ case CLERI_GID_K_LAST:
+ aggr->gid = CLERI_GID_F_LAST;
+ break;
+
+ case CLERI_GID_K_TIMEVAL:
+ aggr->gid = CLERI_GID_F_TIMEVAL;
+ break;
+
+ case CLERI_GID_K_INTERVAL:
+ aggr->gid = CLERI_GID_F_INTERVAL;
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+ }
+
+ VEC_APPEND
+
+ break;
+
+ case CLERI_GID_F_TIMEVAL:
+ case CLERI_GID_F_INTERVAL:
+ AGGR_NEW
+ {
+ aggr->timespan = 1;
+ aggr->group_by = 0;
+ }
+
+ VEC_APPEND
+
+ break;
+ case CLERI_GID_F_FILTER:
+ AGGR_NEW
+ {
+ cleri_node_t * onode;
+ if ( children->node->children->node->children->
+ next->next->next->next == NULL)
+ {
+ aggr->filter_opr = CEXPR_EQ;
+ onode = children->node->children->node->
+ children->next->next->node->
+ children->node;
+ }
+ else
+ {
+ aggr->filter_opr = cexpr_operator_fn(
+ children->node->children->node->
+ children->next->next->node->
+ children->node);
+ onode = children->node->children->node->
+ children->next->next->next->node->
+ children->node;
+ }
+ if (AGGREGATE_init_filter(aggr, onode, err_msg))
+ {
+ AGGREGATE_free(aggr);
+ siridb_aggregate_list_free(vec);
+ return NULL; /* err_msg is set */
+ }
+ }
+
+ VEC_APPEND
+
+ break;
+
+ case CLERI_GID_F_DERIVATIVE:
+ AGGR_NEW
+ {
+ cleri_node_t * dlist = children->node->children->
+ node->children->next->next->node;
+
+ if (dlist->children != NULL && dlist->children->node != NULL)
+ {
+ /* result is at least positive, checked earlier */
+ aggr->timespan =
+ (double) CLERI_NODE_DATA(dlist->children->node);
+
+ if (!aggr->timespan)
+ {
+ sprintf(err_msg,
+ "Time-span must be an integer value "
+ "larger than zero.");
+ AGGREGATE_free(aggr);
+ siridb_aggregate_list_free(vec);
+ return NULL;
+ }
+
+ if (dlist->children->next != NULL)
+ {
+ /* result is always positive */
+ aggr->group_by = CLERI_NODE_DATA(
+ dlist->children->next->next->node);
+
+ if (!aggr->group_by)
+ {
+ sprintf(err_msg,
+ "Group by time must be an integer "
+ "value larger than zero.");
+ AGGREGATE_free(aggr);
+ siridb_aggregate_list_free(vec);
+ return NULL;
+ }
+
+ aggr->timespan /= (double) aggr->group_by;
+ }
+ }
+ }
+
+ VEC_APPEND
+
+ break;
+
+ case CLERI_GID_F_DIFFERENCE:
+ case CLERI_GID_F_COUNT:
+ case CLERI_GID_F_MAX:
+ case CLERI_GID_F_MEAN:
+ case CLERI_GID_F_MEDIAN:
+ case CLERI_GID_F_MEDIAN_HIGH:
+ case CLERI_GID_F_MEDIAN_LOW:
+ case CLERI_GID_F_MIN:
+ case CLERI_GID_F_PVARIANCE:
+ case CLERI_GID_F_SUM:
+ case CLERI_GID_F_VARIANCE:
+ case CLERI_GID_F_STDDEV:
+ case CLERI_GID_F_FIRST:
+ case CLERI_GID_F_LAST:
+ AGGR_NEW
+ if (children->node->children->node->children->
+ next->next->next != NULL)
+ {
+ /* result is always positive, checked earlier */
+ aggr->group_by = CLERI_NODE_DATA(
+ children->node->children->node->
+ children->next->next->node->children->
+ node);
+
+ if (!aggr->group_by)
+ {
+ sprintf(err_msg,
+ "Group by time must be an integer value "
+ "larger than zero.");
+ AGGREGATE_free(aggr);
+ siridb_aggregate_list_free(vec);
+ return NULL;
+ }
+ }
+
+ VEC_APPEND
+
+ break;
+
+ case CLERI_GID_F_ALL:
+ break;
+
+ case CLERI_GID_F_POINTS:
+ log_warning("Keyword 'points' is deprecated, "
+ "use '*' or 'all' instead.");
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ if (children->next == NULL)
+ {
+ break;
+ }
+
+ children = children->next->next;
+ }
+
+ return vec;
+}
+
+/*
+ * Destroy aggregates list. (parsing NULL is not allowed)
+ */
+void siridb_aggregate_list_free(vec_t * alist)
+{
+ size_t i;
+ for (i = 0; i < alist->len; i++)
+ {
+ AGGREGATE_free(alist->data[i]);
+ }
+ free(alist);
+}
+
+/*
+ * Returns 1 (true) if at least one aggregation requires all points to be queried.
+ */
+int siridb_aggregate_can_skip(cleri_children_t * children)
+{
+ cleri_node_t * nd = \
+ children->node->children->node->children->node->children->node;
+
+ switch (nd->cl_obj->gid)
+ {
+ case CLERI_GID_F_COUNT:
+ case CLERI_GID_F_FIRST:
+ case CLERI_GID_F_LAST:
+ return nd->children->next->next->next == NULL;
+
+ default:
+ return 0;
+ }
+}
+
+/*
+ * Return a new allocated points object or the same object as source.
+ * In case of an error NULL is returned and an error message is set.
+ */
+siridb_points_t * siridb_aggregate_run(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg)
+{
+ assert (source->len);
+ if (aggr->limit)
+ {
+ return AGGREGATE_limit(source, aggr, err_msg);
+ }
+
+ if (aggr->group_by)
+ {
+ return AGGREGATE_group_by(source, aggr, err_msg);
+ }
+
+ switch (aggr->gid)
+ {
+ case CLERI_GID_F_DIFFERENCE:
+ return AGGREGATE_difference(source, err_msg);
+
+ case CLERI_GID_F_DERIVATIVE:
+ return AGGREGATE_derivative(source, aggr, err_msg);
+
+ case CLERI_GID_F_FILTER:
+ return AGGREGATE_filter(source, aggr, err_msg);
+
+ case CLERI_GID_F_INTERVAL:
+ return AGGREGATE_interval(source, err_msg);
+
+ case CLERI_GID_F_TIMEVAL:
+ return AGGREGATE_timeval(source, err_msg);
+
+ default:
+ return AGGREGATE_to_one(source, aggr, err_msg);
+ }
+
+ return NULL;
+}
+
+/*
+ * Returns NULL in case an error has occurred.
+ */
+static siridb_aggr_t * AGGREGATE_new(uint32_t gid)
+{
+ siridb_aggr_t * aggr = malloc(sizeof(siridb_aggr_t));
+ if (aggr == NULL)
+ {
+ return NULL;
+ }
+ aggr->gid = gid;
+ aggr->group_by = 0;
+ aggr->limit = 0;
+ aggr->offset = 0;
+ aggr->timespan = 1.0;
+ aggr->regex = NULL;
+ aggr->match_data = NULL;
+ aggr->filter_via.raw = NULL;
+ aggr->filter_tp = TP_INT; /* when string we must cleanup more */
+ return aggr;
+}
+
+/*
+ * Destroy aggregate object. (parsing NULL is not allowed)
+ */
+static void AGGREGATE_free(siridb_aggr_t * aggr)
+{
+ if (aggr->filter_tp == TP_STRING)
+ {
+ free(aggr->filter_via.raw);
+ pcre2_code_free(aggr->regex);
+ pcre2_match_data_free(aggr->match_data);
+ }
+ free(aggr);
+}
+
+/*
+ * Returns 0 if successful or -1 in case or an error.
+ * The err_msg is set for any error.
+ */
+static int AGGREGATE_init_filter(
+ siridb_aggr_t * aggr,
+ cleri_node_t * node,
+ char * err_msg)
+{
+ switch (node->cl_obj->gid)
+ {
+ case CLERI_GID_K_NAN:
+ aggr->filter_tp = TP_DOUBLE;
+ aggr->filter_via.real = NAN;
+ break;
+
+ case CLERI_GID_K_INF:
+ aggr->filter_tp = TP_DOUBLE;
+ aggr->filter_via.real = INFINITY;
+ break;
+
+ case CLERI_GID_K_NINF:
+ aggr->filter_tp = TP_DOUBLE;
+ aggr->filter_via.real = -INFINITY;
+ break;
+
+ case CLERI_GID_R_INTEGER:
+ aggr->filter_tp = TP_INT;
+ aggr->filter_via.int64 = atoll(node->str);
+ break;
+
+ case CLERI_GID_R_FLOAT:
+ aggr->filter_tp = TP_DOUBLE;
+ aggr->filter_via.real = xstr_to_double(node->str);
+ break;
+
+ case CLERI_GID_STRING:
+ aggr->filter_tp = TP_STRING;
+ aggr->filter_via.raw = malloc(node->len - 1);
+ if (aggr->filter_via.raw == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ return -1;
+ }
+ xstr_extract_string(
+ (char *) aggr->filter_via.raw, node->str, node->len);
+ return 0;
+
+ case CLERI_GID_R_REGEX:
+ if (aggr->filter_opr != CEXPR_EQ && aggr->filter_opr != CEXPR_NE)
+ {
+ sprintf(err_msg,
+ "Regular expressions can only be used with 'equal' (==) "
+ "or 'not equal' (!=) operator.");
+ return -1;
+ }
+ aggr->filter_tp = TP_STRING;
+ /* extract and compile regular expression */
+ if (siridb_re_compile(
+ &aggr->regex,
+ &aggr->match_data,
+ node->str,
+ node->len,
+ err_msg))
+ {
+ return -1; /* error_msg is set */
+ }
+ return 0;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ if (aggr->filter_opr == CEXPR_IN || aggr->filter_opr == CEXPR_NI)
+ {
+ sprintf(err_msg,
+ "Operator '%s' can only be used with strings.",
+ (aggr->filter_opr == CEXPR_IN) ? "~" : "!~");
+ return -1;
+ }
+
+ return 0;
+}
+
+static siridb_points_t * AGGREGATE_limit(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg)
+{
+ if (source->len <= aggr->limit)
+ {
+ return source;
+ }
+
+ uint64_t timespan =
+ source->data[source->len - 1].ts - source->data[0].ts;
+
+ aggr->group_by = timespan / aggr->limit + 1;
+ aggr->offset = (source->data[0].ts - 1) % aggr->group_by;
+
+ return AGGREGATE_group_by(source, aggr, err_msg);
+}
+
+static siridb_points_t * AGGREGATE_derivative(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg)
+{
+ if (source->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use derivative() on string type.");
+ return NULL;
+ }
+
+ size_t len = source->len - 1;
+ siridb_points_t * points = siridb_points_new(len, TP_DOUBLE);
+
+ if (points == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ }
+ else
+ {
+ points->len = len;
+
+ if (len)
+ {
+ siridb_point_t * prev = source->data;
+ siridb_point_t * spt;
+ siridb_point_t * dpt;
+ size_t i;
+
+ switch (source->tp)
+ {
+ case TP_INT:
+ for ( i = 0, spt= prev + 1, dpt = points->data;
+ i < len;
+ i++, spt++, dpt++)
+ {
+ dpt->ts = spt->ts;
+ dpt->val.real = ((double) spt->val.int64 - prev->val.int64)
+ / (double) (spt->ts - prev->ts) * aggr->timespan;
+ prev = spt;
+ }
+ break;
+
+ case TP_DOUBLE:
+ for ( i = 0, spt= prev + 1, dpt = points->data;
+ i < len;
+ i++, spt++, dpt++)
+ {
+ dpt->ts = spt->ts;
+ dpt->val.real = (spt->val.real - prev->val.real)
+ / (double) (spt->ts - prev->ts) * aggr->timespan;
+ prev = spt;
+ }
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+ }
+ }
+ return points;
+}
+
+static siridb_points_t * AGGREGATE_difference(
+ siridb_points_t * source,
+ char * err_msg)
+{
+ if (source->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use difference() on string type.");
+ return NULL;
+ }
+
+ size_t len = source->len - 1;
+ siridb_points_t * points = siridb_points_new(len, source->tp);
+
+ if (points == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ }
+ else
+ {
+ points->len = len;
+
+ if (len)
+ {
+ siridb_point_t * prev = source->data;
+ siridb_point_t * spt;
+ siridb_point_t * dpt;
+ size_t i;
+
+ switch (source->tp)
+ {
+ case TP_INT:
+ {
+ int64_t first = prev->val.int64;
+ int64_t last;
+ for ( i = 0, spt= prev + 1, dpt = points->data;
+ i < len;
+ i++, spt++, dpt++)
+ {
+ last = spt->val.int64;
+
+ if ((first > 0 && last < LLONG_MIN + first) ||
+ (first < 0 && last > LLONG_MAX + first))
+ {
+ sprintf(err_msg,
+ "Overflow detected while using difference().");
+ siridb_points_free(points);
+ return NULL;
+ }
+
+ dpt->ts = spt->ts;
+ dpt->val.int64 = last - first;
+
+ first = last;
+ }
+
+ }
+ break;
+
+ case TP_DOUBLE:
+ for ( i = 0, spt= prev + 1, dpt = points->data;
+ i < len;
+ i++, spt++, dpt++)
+ {
+ dpt->ts = spt->ts;
+ dpt->val.real = spt->val.real - prev->val.real;
+ prev = spt;
+ }
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+ }
+ }
+ return points;
+}
+
+static siridb_points_t * AGGREGATE_interval(
+ siridb_points_t * source,
+ char * err_msg)
+{
+ size_t len = source->len - 1;
+ siridb_points_t * points = siridb_points_new(len, TP_INT);
+
+ if (points == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ }
+ else
+ {
+ points->len = len;
+
+ if (len)
+ {
+ siridb_point_t * prev = source->data;
+ siridb_point_t * curr = prev + 1;
+ siridb_point_t * end = source->data + source->len;
+ siridb_point_t * dest = points->data;
+
+ for (; curr < end; ++prev, ++curr, ++dest)
+ {
+ uint64_t diff = curr->ts - prev->ts;
+ if (diff > LLONG_MAX)
+ {
+ sprintf(err_msg,
+ "Overflow detected while using difference().");
+ siridb_points_free(points);
+ return NULL;
+ }
+ dest->ts = curr->ts;
+ dest->val.int64 = (int64_t) diff;
+ }
+ }
+ }
+ return points;
+}
+
+static siridb_points_t * AGGREGATE_timeval(
+ siridb_points_t * source,
+ char * err_msg)
+{
+ siridb_points_t * points = siridb_points_new(source->len, TP_INT);
+
+ if (points == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ }
+ else
+ {
+ siridb_point_t * curr = source->data;
+ siridb_point_t * end = source->data + source->len;
+ siridb_point_t * dest = points->data;
+
+ points->len = source->len;
+
+ for (; curr < end; ++curr, ++dest)
+ {
+ if (curr->ts > LLONG_MAX)
+ {
+ sprintf(err_msg,
+ "Overflow detected while using difference().");
+ siridb_points_free(points);
+ return NULL;
+ }
+ dest->ts = curr->ts;
+ dest->val.int64 = (int64_t) curr->ts;
+ }
+ }
+ return points;
+}
+
+static int AGGREGATE_regex_cmp(siridb_aggr_t * aggr, char * val)
+{
+ int ret;
+ ret = pcre2_match(
+ aggr->regex,
+ (PCRE2_SPTR8) val,
+ strlen(val),
+ 0, /* start looking at this point */
+ 0, /* OPTIONS */
+ aggr->match_data,
+ 0); /* length of sub_str_vec */
+ return aggr->filter_opr == CEXPR_EQ ? ret >= 0 : ret < 0;
+}
+
+static siridb_points_t * AGGREGATE_filter(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg)
+{
+ qp_via_t value = aggr->filter_via;
+
+ if (source->tp != aggr->filter_tp)
+ {
+ if (source->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use a number filter on string type.");
+ return NULL;
+ }
+
+ switch (aggr->filter_tp)
+ {
+ case TP_STRING:
+ sprintf(err_msg, "Cannot use a string filter on number type.");
+ return NULL;
+
+ case TP_INT:
+ value.real = (double) value.int64;
+ break;
+
+ case TP_DOUBLE:
+ value.int64 = (int64_t) value.real;
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+ }
+
+ siridb_points_t * points = siridb_points_new(source->len, source->tp);
+
+ if (points == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ }
+ else
+ {
+ size_t i;
+ siridb_point_t * spt;
+ siridb_point_t * dpt;
+ switch ((points_tp) source->tp)
+ {
+ case TP_STRING:
+ for ( i = 0, spt = source->data, dpt = points->data;
+ i < source->len;
+ i++, spt++)
+ {
+ if (value.str != NULL /* NULL is a regular expression */
+ ? cexpr_str_cmp(
+ aggr->filter_opr,
+ spt->val.str, value.str)
+ : AGGREGATE_regex_cmp(aggr, spt->val.str))
+ {
+ dpt->ts = spt->ts;
+ dpt->val.str = strdup(spt->val.str);
+ if (dpt->val.str == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ siridb_points_free(points);
+ return NULL;
+ }
+ dpt++;
+ }
+ }
+ break;
+
+ case TP_INT:
+ for ( i = 0, spt = source->data, dpt = points->data;
+ i < source->len;
+ i++, spt++)
+ {
+ if (cexpr_int_cmp(
+ aggr->filter_opr,
+ spt->val.int64,
+ value.int64))
+ {
+ dpt->ts = spt->ts;
+ dpt->val = spt->val;
+ dpt++;
+ }
+ }
+ break;
+
+ case TP_DOUBLE:
+ for ( i = 0, spt = source->data, dpt = points->data;
+ i < source->len;
+ i++, spt++)
+ {
+ if (cexpr_double_cmp(
+ aggr->filter_opr,
+ spt->val.real,
+ value.real))
+ {
+ dpt->ts = spt->ts;
+ dpt->val = spt->val;
+ dpt++;
+ }
+ }
+ break;
+
+ default:
+ assert (0);
+ dpt = NULL;
+ break;
+ }
+
+ points->len = dpt - points->data;
+
+ if (source->len > points->len)
+ {
+ dpt = (siridb_point_t *) realloc(
+ points->data,
+ points->len * sizeof(siridb_point_t));
+ if (dpt == NULL && points->len)
+ {
+ /* not critical */
+ log_error("Error while re-allocating memory for points");
+ }
+ else
+ {
+ points->data = dpt;
+ }
+ }
+ }
+
+ return points;
+}
+
+static siridb_points_t * AGGREGATE_to_one(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg)
+{
+ siridb_points_t * points;
+ /* get correct callback function */
+ AGGR_cb aggr_cb = AGGREGATES[aggr->gid - F_OFFSET];
+
+ /* create new points with max possible size after re-indexing */
+ switch(aggr->gid)
+ {
+ case CLERI_GID_F_MEAN:
+ case CLERI_GID_F_MEDIAN:
+ case CLERI_GID_F_PVARIANCE:
+ case CLERI_GID_F_VARIANCE:
+ case CLERI_GID_F_STDDEV:
+ points = siridb_points_new(1, TP_DOUBLE);
+ break;
+ case CLERI_GID_F_COUNT:
+ points = siridb_points_new(1, TP_INT);
+ break;
+ case CLERI_GID_F_MEDIAN_HIGH:
+ case CLERI_GID_F_MAX:
+ case CLERI_GID_F_MEDIAN_LOW:
+ case CLERI_GID_F_MIN:
+ case CLERI_GID_F_SUM:
+ case CLERI_GID_F_FIRST:
+ case CLERI_GID_F_LAST:
+ points = siridb_points_new(1, source->tp);
+ break;
+ default:
+ assert (0);
+ points = NULL;
+ }
+
+ if (points == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ return NULL;
+ }
+
+ /* set time-stamp */
+ points->data->ts = source->data[
+ (aggr->gid == CLERI_GID_F_FIRST) ? 0 : (source->len - 1)].ts;
+
+ /* set value */
+ if (aggr_cb(points->data, source, aggr, err_msg))
+ {
+ /* error occurred, return NULL */
+ siridb_points_free(points);
+ return NULL;
+ }
+
+ points->len++;
+ return points;
+}
+
+static siridb_points_t * AGGREGATE_group_by(
+ siridb_points_t * source,
+ siridb_aggr_t * aggr,
+ char * err_msg)
+{
+ siridb_point_t * point;
+ siridb_points_t * points;
+ siridb_points_t group;
+ uint64_t max_sz;
+ uint64_t goup_ts;
+ size_t start, end;
+
+ group.tp = source->tp;
+
+ max_sz = ((source->data + source->len - 1)->ts - source->data->ts)
+ / aggr->group_by + 2;
+
+ if (max_sz > source->len)
+ {
+ max_sz = source->len;
+ }
+
+ /* get correct callback function */
+ AGGR_cb aggr_cb = AGGREGATES[aggr->gid - F_OFFSET];
+
+ /* create new points with max possible size after re-indexing */
+ switch(aggr->gid)
+ {
+ case CLERI_GID_F_MEAN:
+ case CLERI_GID_F_MEDIAN:
+ case CLERI_GID_F_PVARIANCE:
+ case CLERI_GID_F_VARIANCE:
+ case CLERI_GID_F_STDDEV:
+ case CLERI_GID_F_DERIVATIVE:
+ points = siridb_points_new(max_sz, TP_DOUBLE);
+ break;
+ case CLERI_GID_F_COUNT:
+ case CLERI_GID_F_TIMEVAL:
+ case CLERI_GID_F_INTERVAL:
+ points = siridb_points_new(max_sz, TP_INT);
+ break;
+ case CLERI_GID_F_MEDIAN_HIGH:
+ case CLERI_GID_F_MAX:
+ case CLERI_GID_F_MEDIAN_LOW:
+ case CLERI_GID_F_MIN:
+ case CLERI_GID_F_SUM:
+ case CLERI_GID_F_DIFFERENCE:
+ case CLERI_GID_F_FIRST:
+ case CLERI_GID_F_LAST:
+ points = siridb_points_new(max_sz, group.tp);
+ break;
+ default:
+ assert (0);
+ points = NULL;
+ }
+
+ if (points == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ return NULL;
+ }
+
+ goup_ts = GROUP_TS(source->data);
+
+ for(start = end = 0; end < source->len; end++)
+ {
+ if ((source->data + end)->ts > goup_ts)
+ {
+ group.data = (source->data + start);
+ group.len = end - start;
+ point = points->data + points->len;
+ point->ts = goup_ts;
+ if (aggr_cb(point, &group, aggr, err_msg))
+ {
+ /* error occurred, return NULL */
+ siridb_points_free(points);
+ return NULL;
+ }
+ points->len++;
+ start = end;
+ goup_ts = GROUP_TS((source->data + end));
+ }
+ }
+
+ group.data = (source->data + start);
+ group.len = end - start;
+ point = points->data + points->len;
+ point->ts = goup_ts;
+ if (aggr_cb(point, &group, aggr, err_msg))
+ {
+ /* error occurred, return NULL */
+ siridb_points_free(points);
+ return NULL;
+ }
+ points->len++;
+
+ if (points->len < max_sz)
+ {
+ /* shrink points allocation */
+ point = realloc(points->data, points->len * sizeof(siridb_point_t));
+ if (point == NULL && points->len)
+ {
+ /* not critical */
+ log_error("Re-allocation points failed.");
+ }
+ else
+ {
+ points->data = point;
+ }
+ }
+ /* else { assert (points->len == max_sz); } */
+
+ return points;
+}
+
+static int aggr_count(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg __attribute__((unused)))
+{
+ point->val.int64 = points->len;
+ return 0;
+}
+
+static int aggr_derivative(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr,
+ char * err_msg)
+{
+ assert (points->len);
+
+ if (points->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use derivative() on string type.");
+ return -1;
+ }
+
+ if (points->len == 1)
+ {
+ point->val.real = 0.0;
+ }
+ else
+ {
+ double first, last;
+
+ switch (points->tp)
+ {
+ case TP_INT:
+ first = (double) points->data->val.int64;
+ last = (double) (points->data + points->len - 1)->val.int64;
+ break;
+ case TP_DOUBLE:
+ first = points->data->val.real;
+ last = (points->data + points->len - 1)->val.real;
+ break;
+ default:
+ assert (0);
+ first = 0.0;
+ last = 0.0;
+ break;
+ }
+
+ /* time-span is actually a factor when used with group_by */
+ point->val.real = (last- first) * aggr->timespan;
+ }
+
+ return 0;
+}
+
+static int aggr_difference(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ switch (points->tp)
+ {
+ case TP_STRING:
+ sprintf(err_msg, "Cannot use difference() on string type.");
+ return -1;
+
+ case TP_INT:
+ if (points->len == 1)
+ {
+ point->val.int64 = 0;
+ }
+ else
+ {
+ int64_t first = points->data->val.int64;
+ int64_t last = (points->data + points->len - 1)->val.int64;
+
+ if ((first > 0 && last < LLONG_MIN + first) ||
+ (first < 0 && last > LLONG_MAX + first))
+ {
+ sprintf(err_msg, "Overflow detected while using difference().");
+ return -1;
+ }
+
+ point->val.int64 = last- first;
+ }
+ break;
+
+ case TP_DOUBLE:
+ if (points->len == 1)
+ {
+ point->val.real = 0.0;
+ }
+ else
+ {
+ point->val.real =
+ (points->data + points->len - 1)->val.real -
+ points->data->val.real;
+ }
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ return 0;
+}
+
+static int aggr_max(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ if (points->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use max() on string type.");
+ return -1;
+ }
+
+ if (points->tp == TP_INT)
+ {
+ int64_t max = points->data->val.int64;
+ size_t i;
+ for (i = 1; i < points->len; i++)
+ {
+ if ((points->data + i)->val.int64 > max)
+ {
+ max = (points->data + i)->val.int64;
+ }
+ }
+ point->val.int64 = max;
+ }
+ else
+ {
+ double max = points->data->val.real;
+ size_t i;
+ for (i = 1; i < points->len; i++)
+ {
+ if ((points->data + i)->val.real > max)
+ {
+ max = (points->data + i)->val.real;
+ }
+ }
+ point->val.real = max;
+ }
+
+ return 0;
+}
+
+static int aggr_mean(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ double sum = 0.0;
+ size_t i;
+
+ switch (points->tp)
+ {
+ case TP_STRING:
+ sprintf(err_msg, "Cannot use mean() on string type.");
+ return -1;
+
+ case TP_INT:
+ for (i = 0; i < points->len; i++)
+ {
+ sum += (points->data + i)->val.int64;
+ }
+ break;
+
+ case TP_DOUBLE:
+ for (i = 0; i < points->len; i++)
+ {
+ sum += (points->data + i)->val.real;
+ }
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ point->val.real = sum / points->len;
+
+ return 0;
+}
+
+static int aggr_median(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ if (points->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use median() on string type.");
+ return -1;
+ }
+
+ if (points->len == 1)
+ {
+ if (points->tp == TP_INT)
+ {
+ point->val.real = (double) points->data->val.int64;
+ }
+ else
+ {
+ point->val.real = points->data->val.real;
+ }
+ }
+ else if (points->len % 2 == 1)
+ {
+ siridb_median_find_n(point, points, points->len / 2);
+ if (points->tp == TP_INT)
+ {
+ point->val.real = (double) point->val.int64;
+ }
+ }
+ else if (siridb_median_real(point, points, 0.5))
+ {
+ sprintf(err_msg, "Memory allocation error in median.");
+ return -1;
+ }
+
+ return 0;
+}
+
+static int aggr_median_high(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ if (points->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use median_high() on string type.");
+ return -1;
+ }
+
+ if (points->len == 1)
+ {
+ if (points->tp == TP_INT)
+ {
+ point->val.int64 = points->data->val.int64;
+ }
+ else
+ {
+ point->val.real = points->data->val.real;
+ }
+ }
+ else if (siridb_median_find_n(point, points, points->len / 2))
+ {
+ sprintf(err_msg, "Memory allocation error in median high.");
+ return -1;
+ }
+ return 0;
+}
+
+static int aggr_median_low(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ if (points->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use median_low() on string type.");
+ return -1;
+ }
+
+ if (points->len == 1)
+ {
+ if (points->tp == TP_INT)
+ {
+ point->val.int64 = points->data->val.int64;
+ }
+ else
+ {
+ point->val.real = points->data->val.real;
+ }
+ }
+ else if (siridb_median_find_n(point, points, (points->len - 1) / 2))
+ {
+ sprintf(err_msg, "Memory allocation error in median low.");
+ return -1;
+ }
+ return 0;
+}
+
+static int aggr_min(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ if (points->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot use min() on string type.");
+ return -1;
+ }
+
+ if (points->tp == TP_INT)
+ {
+ int64_t min = points->data->val.int64;
+ size_t i;
+ for (i = 1; i < points->len; i++)
+ {
+ if ((points->data + i)->val.int64 < min)
+ {
+ min = (points->data + i)->val.int64;
+ }
+ }
+ point->val.int64 = min;
+ }
+ else
+ {
+ double min = points->data->val.real;
+ size_t i;
+ for (i = 1; i < points->len; i++)
+ {
+ if ((points->data + i)->val.real < min)
+ {
+ min = (points->data + i)->val.real;
+ }
+ }
+ point->val.real = min;
+ }
+
+ return 0;
+}
+
+static int aggr_pvariance(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ switch (points->tp)
+ {
+ case TP_STRING:
+ sprintf(err_msg, "Cannot use pvariance() on string type.");
+ return -1;
+
+ case TP_INT:
+ case TP_DOUBLE:
+ point->val.real = siridb_variance(points) / points->len;
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ return 0;
+}
+
+static int aggr_sum(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ switch (points->tp)
+ {
+ case TP_STRING:
+ sprintf(err_msg, "Cannot use sum() on string type.");
+ return -1;
+
+ case TP_INT:
+ {
+ int64_t sum = 0;
+ int64_t tmp;
+ size_t i;
+ for (i = 0; i < points->len; i++)
+ {
+ tmp = (points->data + i)->val.int64;
+ if ((tmp > 0 && sum > LLONG_MAX - tmp) ||
+ (tmp < 0 && sum < LLONG_MIN - tmp))
+ {
+ sprintf(err_msg, "Overflow detected while using sum().");
+ return -1;
+ }
+ sum += tmp;
+ }
+ point->val.int64 = sum;
+ }
+ break;
+
+ case TP_DOUBLE:
+ {
+ double sum = 0.0;
+ size_t i;
+ for (i = 0; i < points->len; i++)
+ {
+ sum += (points->data + i)->val.real;
+ }
+ point->val.real = sum;
+ }
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ return 0;
+}
+
+static int aggr_variance(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ switch (points->tp)
+ {
+ case TP_STRING:
+ sprintf(err_msg, "Cannot use variance() on string type.");
+ return -1;
+
+ case TP_INT:
+ case TP_DOUBLE:
+ point->val.real = (points->len > 1) ?
+ siridb_variance(points) / (points->len - 1) : 0.0;
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ return 0;
+}
+
+static int aggr_stddev(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg)
+{
+ assert (points->len);
+
+ switch (points->tp)
+ {
+ case TP_STRING:
+ sprintf(err_msg, "Cannot use stddev() on string type.");
+ return -1;
+
+ case TP_INT:
+ case TP_DOUBLE:
+ point->val.real = (points->len > 1) ?
+ sqrt(siridb_variance(points) / (points->len - 1)) : 0.0;
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ return 0;
+}
+
+static int aggr_first(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg __attribute__((unused)))
+{
+ assert (points->len);
+
+ siridb_point_t * source = points->data;
+
+ switch (points->tp)
+ {
+ case TP_STRING:
+ point->ts = source->ts;
+ point->val.str = strdup(source->val.str);
+ if (point->val.str == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ return -1;
+ }
+ break;
+
+ case TP_INT:
+ case TP_DOUBLE:
+ point->val = source->val;
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ return 0;
+}
+
+static int aggr_last(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ siridb_aggr_t * aggr __attribute__((unused)),
+ char * err_msg __attribute__((unused)))
+{
+ assert (points->len);
+
+ siridb_point_t * source = points->data + (points->len - 1);
+
+ switch (points->tp)
+ {
+ case TP_STRING:
+ point->ts = source->ts;
+ point->val.str = strdup(source->val.str);
+ if (point->val.str == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ return -1;
+ }
+ break;
+
+ case TP_INT:
+ case TP_DOUBLE:
+ point->val = source->val;
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ return 0;
+}
--- /dev/null
+/*
+ * auth.c - Handle SiriDB authentication.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/auth.h>
+#include <siri/db/db.h>
+#include <siri/db/servers.h>
+#include <siri/db/users.h>
+#include <siri/net/protocol.h>
+#include <siri/net/stream.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <stdlib.h>
+#include <string.h>
+
+cproto_server_t siridb_auth_user_request(
+ sirinet_stream_t * client,
+ qp_obj_t * qp_username,
+ qp_obj_t * qp_password,
+ qp_obj_t * qp_dbname)
+{
+ siridb_t * siridb;
+ siridb_user_t * user;
+
+ char username[qp_username->len + 1];
+ memcpy(username, qp_username->via.raw, qp_username->len);
+ username[qp_username->len] = 0;
+
+ char password[qp_password->len + 1];
+ memcpy(password, qp_password->via.raw, qp_password->len);
+ password[qp_password->len] = 0;
+
+ if ((siridb = siridb_get_by_qp(siri.siridb_list, qp_dbname)) == NULL)
+ {
+ log_warning("User authentication request failed: unknown database");
+ return CPROTO_ERR_AUTH_UNKNOWN_DB;
+ }
+
+ if ((user = siridb_users_get_user(
+ siridb,
+ username,
+ password)) == NULL)
+ {
+ if (strcmp(username, "sa") == 0)
+ {
+ log_warning(
+ "User authentication request failed: "
+ "invalid credentials for user `sa`, "
+ "did you mean to use the default database user `iris`?");
+ }
+ else
+ {
+ log_warning(
+ "User authentication request failed: invalid credentials");
+ }
+ return CPROTO_ERR_AUTH_CREDENTIALS;
+ }
+
+ siridb_incref(siridb);
+ if (client->siridb)
+ {
+ siridb_decref(client->siridb);
+ }
+
+ siridb_user_incref(user);
+ if (client->origin)
+ {
+ siridb_user_decref(((siridb_user_t *) client->origin));
+ }
+
+ client->siridb = siridb;
+ client->origin = user;
+
+ return CPROTO_RES_AUTH_SUCCESS;
+}
+
+/*
+ * Note: qp_version, qp_dbname, qp_min_version must by or type raw and must be
+ * null terminated.
+ */
+bproto_server_t siridb_auth_server_request(
+ sirinet_stream_t * client,
+ qp_obj_t * qp_uuid,
+ qp_obj_t * qp_dbname,
+ qp_obj_t * qp_version,
+ qp_obj_t * qp_min_version)
+{
+ siridb_t * siridb;
+ siridb_server_t * server;
+ uuid_t uuid;
+
+ if (qp_uuid->len != 16)
+ {
+ return BPROTO_AUTH_ERR_INVALID_UUID;
+ }
+
+ if (siri_version_cmp(
+ (const char *) qp_version->via.raw, SIRIDB_MINIMAL_VERSION) < 0)
+ {
+ return BPROTO_AUTH_ERR_VERSION_TOO_OLD;
+ }
+
+ if (siri_version_cmp(
+ (const char *) qp_min_version->via.raw, SIRIDB_VERSION) > 0)
+ {
+ return BPROTO_AUTH_ERR_VERSION_TOO_NEW;
+ }
+
+ memcpy(uuid, qp_uuid->via.raw, 16);
+
+ if ((siridb = siridb_get(
+ siri.siridb_list,
+ (const char *) qp_dbname->via.raw)) == NULL)
+ {
+ return BPROTO_AUTH_ERR_UNKNOWN_DBNAME;
+ }
+
+ if ( (server = siridb_servers_by_uuid(siridb->servers, uuid)) == NULL ||
+ server == siridb->server)
+ {
+ /*
+ * Respond with unknown uuid when not found or in case its 'this'
+ * server.
+ */
+ return BPROTO_AUTH_ERR_UNKNOWN_UUID;
+ }
+
+ siridb_incref(siridb);
+ if (client->siridb)
+ {
+ siridb_decref(client->siridb);
+ }
+
+ client->siridb = siridb;
+ client->origin = server;
+
+ free(server->version);
+ server->version = strdup((const char *) qp_version->via.raw);
+
+ /* we must increment the server reference counter */
+ siridb_server_incref(server);
+
+ return BPROTO_AUTH_SUCCESS;
+}
--- /dev/null
+/*
+ * buffer.c - Buffer for integer and double values.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <logger/logger.h>
+#include <siri/db/buffer.h>
+#include <siri/db/db.h>
+#include <siri/db/misc.h>
+#include <siri/db/shard.h>
+#include <siri/db/shards.h>
+#include <siri/siri.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <xpath/xpath.h>
+#include <assert.h>
+
+#define SIRIDB_BUFFER_FN "buffer.dat"
+
+/* when set to 1, no caching is done. 1 is the minimum value. */
+#define SIRIDB_BUFFER_CACHE 64
+
+static int buffer__create_new(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series);
+static int buffer__use_empty(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series);
+static void buffer__migrate_to_new(char * pt, size_t sz);
+static void buffer__init_template(char * template, size_t size);
+
+
+/* buffer__start cannot conflict with a series_id since id 0 is never used */
+static const uint32_t buffer__start = 0x00000000;
+static const uint64_t buffer__end = 0xffffffffffffffff;
+
+
+siridb_buffer_t * siridb_buffer_new(void)
+{
+ siridb_buffer_t * buffer = malloc(sizeof(siridb_buffer_t));
+ if (buffer == NULL)
+ {
+ return NULL;
+ }
+ buffer->empty = vec_new(VEC_DEFAULT_SIZE);
+ if (buffer->empty == NULL)
+ {
+ free(buffer);
+ return NULL;
+ }
+ buffer->fd = 0;
+ buffer->fp = NULL;
+ buffer->len = 0;
+ buffer->_to_size = 0; /* 0 means no new size */
+ buffer->path = NULL;
+ buffer->size = 0;
+ buffer->template = NULL;
+
+ return buffer;
+}
+
+void siridb_buffer_free(siridb_buffer_t * buffer)
+{
+ if (buffer->fp != NULL)
+ {
+ fclose(buffer->fp);
+ }
+ free(buffer->template);
+ free(buffer->path);
+ vec_free(buffer->empty);
+ free(buffer);
+}
+
+void siridb_buffer_close(siridb_buffer_t * buffer)
+{
+ if (buffer->fp != NULL)
+ {
+ fclose(buffer->fp);
+ buffer->fp = NULL;
+ }
+}
+
+_Bool siridb_buffer_is_valid_size(ssize_t ssize)
+{
+ return ssize >= 512 && (ssize % 512) == 0 && ssize <= MAX_BUFFER_SZ;
+}
+
+void siridb_buffer_set_path(siridb_buffer_t * buffer, const char * str)
+{
+ size_t lstr = strlen(str);
+ size_t lf = strlen(SIRIDB_BUFFER_FN); // size of "buffer.dat"
+ size_t lspf = strlen("/" SIRIDB_BUFFER_FN); // size of "/buffer.dat"
+ assert (buffer->path == NULL);
+
+ if (str[lstr-1] == '/')
+ {
+ buffer->path = strdup(str);
+ }
+ else if (lstr >= lspf && strcmp(str+lstr-lspf, "/" SIRIDB_BUFFER_FN) == 0)
+ {
+ buffer->path = strndup(str, lstr-lf);
+ }
+ else if (asprintf(&buffer->path, "%s/", str) < 0)
+ {
+ free(buffer->path);
+ buffer->path = NULL;
+ }
+
+ if (buffer->path == NULL)
+ {
+ log_critical("Allocation error while setting buffer path");
+ return;
+ }
+}
+
+int siridb_buffer_test_path(siridb_t * siridb)
+{
+ siridb_misc_get_fn(fn, siridb->buffer->path, SIRIDB_BUFFER_FN)
+ if (siridb->series_map->len && !xpath_file_exist(fn))
+ {
+ log_critical("Cannot read buffer file: '%s'", fn);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * Returns 0 if success or EOF in case of an error.
+ */
+int siridb_buffer_write_empty(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series)
+{
+ memcpy(buffer->template + 4, &series->id, sizeof(uint32_t));
+ return (
+ /* go to the series position in buffer */
+ fseeko( buffer->fp,
+ series->bf_offset,
+ SEEK_SET) ||
+
+ /* write end ts */
+ fwrite( buffer->template,
+ buffer->size,
+ 1,
+ buffer->fp) != 1) ? EOF : 0;
+}
+
+/*
+ * Waring: we must check if the new point fits inside the buffer before using
+ * the 'siridb_buffer_write_point()' function.
+ *
+ * Returns 0 if success or EOF in case of an error.
+ */
+int siridb_buffer_write_point(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series,
+ uint64_t * ts,
+ qp_via_t * val)
+{
+ const size_t sz = sizeof(uint64_t) + sizeof(qp_via_t);
+ char buf[sz];
+
+ ssize_t last_idx = series->buffer->len - 1;
+ assert (last_idx >= 0);
+ memcpy(buf, ts, sizeof(uint64_t));
+ memcpy(buf + sizeof(uint64_t), val, sizeof(qp_via_t));
+
+ return (
+ /* jump to position where to write the new point */
+ fseeko( buffer->fp,
+ series->bf_offset + 8 + (16 * last_idx),
+ SEEK_SET) ||
+
+ /* write time-stamp and value */
+ fwrite(buf, sz, 1, buffer->fp) != 1) ? EOF : 0;
+}
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int siridb_buffer_new_series(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series)
+{
+ /* allocate new buffer */
+ series->buffer = siridb_points_new(buffer->len, series->tp);
+ if (series->buffer == NULL)
+ {
+ /* TODO: maybe we can remove the ERR_ALLOC */
+ ERR_ALLOC
+ return -1;
+ }
+
+ return (buffer->empty->len) ?
+ buffer__use_empty(buffer, series) :
+ buffer__create_new(buffer, series);
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ */
+int siridb_buffer_open(siridb_buffer_t * buffer)
+{
+ int rc;
+ siridb_misc_get_fn(fn, buffer->path, SIRIDB_BUFFER_FN)
+
+ if ((buffer->fp = fopen(fn, "r+")) == NULL)
+ {
+ log_critical("Cannot open '%s' for reading and writing", fn);
+ return -1;
+ }
+
+ buffer->fd = fileno(buffer->fp);
+
+ if (buffer->fd == -1)
+ {
+ log_critical("Cannot get file descriptor: '%s'", fn);
+ fclose(buffer->fp);
+ buffer->fp = NULL;
+ return -1;
+ }
+
+#ifdef __APPLE__
+ rc = 0; /* no posix_fadvise on apple */
+#else
+ const int flags = POSIX_FADV_RANDOM | POSIX_FADV_DONTNEED;
+ rc = posix_fadvise(buffer->fd, 0, 0, flags);
+ if (rc)
+ {
+ log_warning("Cannot set advice for file access: '%s' (%d)", fn, rc);
+ }
+#endif
+
+ return rc;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * (signal might be raised)
+ */
+int siridb_buffer_load(siridb_t * siridb)
+{
+ siridb_buffer_t * buffer = siridb->buffer;
+ FILE * fp;
+ FILE * fp_temp;
+ size_t cur_size = buffer->size;
+ size_t cur_len = cur_size / sizeof(siridb_point_t);
+ size_t new_size = buffer->_to_size ? buffer->_to_size : cur_size;
+ size_t new_len = new_size / sizeof(siridb_point_t);
+ size_t read_at_once = (size_t) (MAX_BUFFER_SZ / cur_size);
+ size_t max_len = cur_len > new_len ? cur_len : new_len;
+ size_t num, i;
+ char * buf, * pt;
+ long int offset = 0;
+ siridb_series_t * series;
+ _Bool log_migrate = 1;
+ uint32_t buf_start, series_id;
+ uint64_t * ts;
+
+ log_info("Loading and cleanup buffer");
+
+ /* we can already set the new buffer size */
+ buffer->size = new_size;
+ buffer->len = new_len;
+
+ buf = malloc(read_at_once * cur_size);
+ buffer->template = malloc(new_size);
+ if (buf == NULL || buffer->template == NULL)
+ {
+ free(buf); /* buffer->template will be cleaned */
+ log_critical("Allocation error while loading buffer");
+ return -1;
+ }
+
+ if (new_size != cur_size)
+ {
+ log_warning(
+ "Changing buffer size from %zu to %zu", cur_size, new_size);
+ }
+
+ buffer__init_template(buffer->template, new_size);
+
+ siridb_misc_get_fn(fn, buffer->path, SIRIDB_BUFFER_FN)
+ siridb_misc_get_fn(fn_temp, buffer->path, "__" SIRIDB_BUFFER_FN)
+
+ if (xpath_file_exist(fn_temp))
+ {
+ free(buf);
+ log_error(
+ "Temporary buffer file found: '%s'. "
+ "Check if something went wrong or remove this file", fn_temp);
+ return -1;
+ }
+
+ if ((fp = fopen(fn, "r")) == NULL)
+ {
+ free(buf);
+ log_info("Buffer file '%s' not found, create a new one.", fn);
+ if ((fp = fopen(fn, "w")) == NULL)
+ {
+ log_critical("Cannot create buffer file '%s'.", fn);
+ return -1;
+ }
+ return fclose(fp);
+ }
+
+ if ((fp_temp = fopen(fn_temp, "w")) == NULL)
+ {
+ log_critical("Cannot open '%s' for writing", fn_temp);
+ fclose(fp);
+ free(buf);
+ return -1;
+ }
+
+ while ((num = fread(buf, cur_size, read_at_once, fp)))
+ {
+ for (i = 0; i < num; i++)
+ {
+ pt = buf + i * cur_size;
+
+ buf_start = *((uint32_t *) pt);
+ if (buf_start != buffer__start)
+ {
+ if (log_migrate)
+ {
+ log_warning("Buffer will be migrated");
+ log_migrate = 0;
+ }
+ buffer__migrate_to_new(pt, cur_size);
+ }
+
+ pt += sizeof(uint32_t);
+ series_id = *((uint32_t *) pt);
+ pt += sizeof(uint32_t);
+
+ series = imap_get(siridb->series_map, series_id);
+
+ if (series == NULL)
+ {
+ continue;
+ }
+ else if (series->tp == TP_STRING)
+ {
+ log_error("Unexpected buffer found for string series '%s'",
+ series->name);
+ continue;
+ }
+
+ series->buffer = siridb_points_new(max_len, series->tp);
+ if (series->buffer == NULL)
+ {
+ log_critical("Cannot allocate a buffer for series id %u",
+ series->id);
+ goto failed;
+ }
+
+ series->bf_offset = offset;
+
+ for (; *(ts = (uint64_t *) pt) != buffer__end; pt += 16)
+ {
+ qp_via_t * val = (qp_via_t *) (pt + 8);
+ siridb_points_add_point(series->buffer, ts, val);
+ }
+
+ offset += new_size;
+ series->length += series->buffer->len;
+
+ pt = buf + i * cur_size;
+ if (new_size > cur_size)
+ {
+ memcpy(buffer->template, pt, cur_size);
+ pt = buffer->template;
+ }
+ else if (new_size < cur_size)
+ {
+ if (series->buffer->len >= new_len)
+ {
+ if (siridb_shards_add_points(
+ siridb,
+ series,
+ series->buffer))
+ {
+ log_critical("Error while sharding points");
+ goto failed;
+ }
+ series->buffer->len = 0;
+ memcpy(
+ buffer->template + 4,
+ &series->id,
+ sizeof(uint32_t));
+ pt = buffer->template;
+ }
+
+ if (siridb_points_resize(series->buffer, new_len))
+ {
+ log_critical("Allocation error while resizing points");
+ goto failed;
+ }
+ }
+
+ /* write to output file and check if write was successful */
+ if ((fwrite(pt, new_size, 1, fp_temp) != 1))
+ {
+ log_critical("Could not write to temporary buffer file: '%s'",
+ fn_temp);
+ goto failed;
+ }
+ }
+ }
+
+ if (new_size != cur_size)
+ {
+ if (siridb_save(siridb))
+ {
+ log_critical("Cannot save changes to SiriDB (database.dat)");
+ goto failed;
+ }
+ buffer__init_template(buffer->template, new_size);
+ }
+
+ free(buf);
+ if (fclose(fp) ||
+ fclose(fp_temp) ||
+ rename(fn_temp, fn))
+ {
+ log_critical("Could not rename '%s' to '%s'.", fn_temp, fn);
+ return -1;
+ }
+
+ return 0;
+
+failed:
+ fclose(fp);
+ fclose(fp_temp);
+ free(buf);
+ return -1;
+}
+
+/*
+ * Reserve a space in the buffer for a new series. The position of this space
+ * in the buffer is read from siridb->empty_buffers so this list must have
+ * at least on spot available.
+ *
+ * Returns 0 if successful or -1 and a signal is raised in case of an error.
+ *
+ * Note that an available spot must be checked before calling this function.
+ * This functions has undefined behavior if no spot is found.
+ */
+static int buffer__use_empty(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series)
+{
+ series->bf_offset = (long int) vec_pop(buffer->empty);
+
+ if (siridb_buffer_write_empty(buffer, series))
+ {
+ ERR_FILE
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Create new space in the buffer and use one position for the new series.
+ * The number of positions that will be allocated is defined by
+ * SIRIDB_BUFFER_CACHE and must be at least one to hold the new series.
+ *
+ * Returns 0 if successful or -1 and a signal is raised in case of an error.
+ */
+static int buffer__create_new(
+ siridb_buffer_t * buffer,
+ siridb_series_t * series)
+{
+ long int buffer_pos;
+
+
+ /* jump to end of buffer */
+ if (fseeko(buffer->fp, 0, SEEK_END))
+ {
+ ERR_FILE
+ return -1;
+ }
+
+ /* bind the current offset to the new series */
+ if ((series->bf_offset = ftello(buffer->fp)) == -1)
+ {
+ ERR_FILE
+ return -1;
+ }
+
+ /* write buffer start and series ID to buffer */
+ if (siridb_buffer_write_empty(buffer, series))
+ {
+ ERR_FILE
+ return -1;
+ }
+
+ buffer_pos = series->bf_offset + buffer->size * SIRIDB_BUFFER_CACHE;
+
+ /* fill buffer with zeros if possible */
+ if (ftruncate(buffer->fd, buffer_pos))
+ {
+ ERR_FILE
+ return -1;
+ }
+
+ /* commit changes to disk */
+ if (fsync(buffer->fd))
+ {
+ ERR_FILE
+ return -1;
+ }
+
+ while ((buffer_pos -= buffer->size) > series->bf_offset)
+ {
+ vec_append_safe(&buffer->empty, (void *) buffer_pos);
+ }
+
+ return 0;
+}
+
+static void buffer__migrate_to_new(char * pt, size_t sz)
+{
+ char * npt = pt;
+ char * end = pt + sz;
+ uint32_t series_id = *((uint32_t *) pt);
+ pt += sizeof(uint32_t);
+ size_t num = *((size_t *) pt);
+ pt += sizeof(size_t);
+
+ memcpy(npt, &buffer__start, sizeof(uint32_t));
+ npt += sizeof(uint32_t);
+ memcpy(npt, &series_id, sizeof(uint32_t));
+ npt += sizeof(uint32_t);
+ memmove(npt, pt, num * 16);
+ npt += num * 16;
+
+ for (; npt < end; npt += sizeof(uint64_t))
+ {
+ memcpy(npt, &buffer__end, sizeof(uint64_t));
+ }
+}
+
+static void buffer__init_template(char * template, size_t size)
+{
+ char * pt, * end;
+ for ( pt = template,
+ end = template + size;
+ pt < end;
+ pt += sizeof(uint64_t))
+ {
+ memcpy(pt, &buffer__end, sizeof(uint64_t));
+ }
+ memcpy(template, &buffer__start, sizeof(uint32_t));
+}
--- /dev/null
+/*
+ * db.c - SiriDB database.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <cfgparser/cfgparser.h>
+#include <lock/lock.h>
+#include <lock/lock.h>
+#include <logger/logger.h>
+#include <math.h>
+#include <procinfo/procinfo.h>
+#include <siri/db/db.h>
+#include <siri/db/misc.h>
+#include <siri/db/series.h>
+#include <siri/db/servers.h>
+#include <siri/db/shard.h>
+#include <siri/db/shards.h>
+#include <siri/db/time.h>
+#include <siri/db/users.h>
+#include <siri/err.h>
+#include <siri/siri.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <uuid/uuid.h>
+#include <xpath/xpath.h>
+#include <vec/vec.h>
+#include <timeit/timeit.h>
+
+/*
+ * database.dat
+ *
+ * SCHEMA -> SIRIDB_SHEMA
+ * UUID -> UUID for 'this' server
+ * DBNAME -> Database name
+ *
+ *
+ */
+
+static siridb_t * siridb__new(void);
+static int siridb__from_unpacker(
+ qp_unpacker_t * unpacker,
+ siridb_t ** siridb,
+ const char * dbpath,
+ char * err_msg);
+static siridb_t * siridb__from_dat(const char * dbpath);
+static int siridb__read_conf(siridb_t * siridb);
+static int siridb__lock(const char * dbpath, int lock_flags);
+
+#define READ_DB_EXIT_WITH_ERROR(ERROR_MSG) \
+ strcpy(err_msg, ERROR_MSG); \
+ siridb__free(*siridb); \
+ *siridb = NULL; \
+ return -1;
+
+/*
+ * Returns the up-time in seconds.
+ */
+int32_t siridb_get_uptime(siridb_t * siridb)
+{
+ return (int32_t) timeit_get(&siridb->start_time);
+}
+
+/*
+ * Returns a value in the range 0 and 100 representing how much percent sine
+ * up-time is idle time.
+ */
+int8_t siridb_get_idle_percentage(siridb_t * siridb)
+{
+ double uptime = (double) siridb_get_uptime(siridb);
+ int8_t idle = (uptime)
+ ? (int8_t) round(siridb->tasks.idle_time / uptime * 100.0f)
+ : 0;
+ /* idle time can technically be larger since we start a database before we
+ * mark the server as started.
+ */
+ return (idle > 100) ? 100 : idle;
+}
+
+/*
+ * Check if at least database.conf and database.dat exist in the path.
+ */
+int siridb_is_db_path(const char * dbpath)
+{
+ char buffer[XPATH_MAX];
+ buffer[XPATH_MAX-1] = '\0';
+ snprintf(buffer,
+ XPATH_MAX-1,
+ "%sdatabase.conf",
+ dbpath);
+ if (!xpath_file_exist(buffer))
+ {
+ return 0; /* false */
+ }
+ snprintf(buffer,
+ XPATH_MAX-1,
+ "%sdatabase.dat",
+ dbpath);
+ if (!xpath_file_exist(buffer))
+ {
+ return 0; /* false */
+ }
+ return 1; /* true */
+}
+
+/*
+ * Returns a siridb object or NULL in case of an error.
+ *
+ * (lock_flags are simple parsed to the lock function)
+ *
+ */
+siridb_t * siridb_new(const char * dbpath, int lock_flags)
+{
+ size_t len = strlen(dbpath);
+ siridb_t * siridb;
+ size_t i;
+
+ if (!len || dbpath[len - 1] != '/')
+ {
+ log_error("Database path should end with a slash. (got: '%s')",
+ dbpath);
+ return NULL;
+ }
+
+ if (!xpath_is_dir(dbpath))
+ {
+ log_error("Cannot find database path '%s'", dbpath);
+ return NULL;
+ }
+
+ if (siridb__lock(dbpath, lock_flags))
+ {
+ log_error("Cannot lock database path '%s'", dbpath);
+ return NULL;
+ }
+
+ siridb = siridb__from_dat(dbpath);
+ if (siridb == NULL)
+ {
+ log_error("Cannot load SiriDB from database path '%s'", dbpath);
+ return NULL;
+ }
+
+ log_info("Start loading database: '%s'", siridb->dbname);
+
+ /* read database.conf */
+ if (siridb__read_conf(siridb))
+ {
+ log_error("Could not read config for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* load users */
+ if (siridb_users_load(siridb))
+ {
+ log_error("Could not read users for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* load servers */
+ if (siridb_servers_load(siridb))
+ {
+ log_error("Could not read servers for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* load series */
+ if (siridb_series_load(siridb))
+ {
+ log_error("Could not read series for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* test buffer path */
+ if (siridb_buffer_test_path(siridb))
+ {
+ log_error("Cannot read buffer for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* load shards */
+ if (siridb_shards_load(siridb))
+ {
+ log_error("Could not read shards for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* load buffer */
+ if (siridb_buffer_load(siridb))
+ {
+ log_error("Could not read buffer for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* open buffer */
+ if (siridb_buffer_open(siridb->buffer))
+ {
+ log_error("Could not open buffer for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* load groups */
+ if (siridb_groups_init(siridb))
+ {
+ log_error("Cannot read groups for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* load tags */
+ if (siridb_tags_init(siridb))
+ {
+ log_error("Cannot read tags for database '%s'", siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ /* update series props */
+ log_info("Updating series properties");
+
+ /* create a copy since 'siridb_series_update_props' might drop a series */
+ vec_t * vec = imap_2vec(siridb->series_map);
+
+ if (vec == NULL)
+ {
+ log_error("Could update series properties for database '%s'",
+ siridb->dbname);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ for (i = 0; i < vec->len; i++)
+ {
+ siridb_series_update_props(siridb, (siridb_series_t * )vec->data[i]);
+ }
+
+ vec_free(vec);
+
+ /* generate pools, this can raise a signal */
+ log_info("Initialize pools");
+ siridb_pools_init(siridb);
+
+ if (!siri_err)
+ {
+ siridb->reindex = siridb_reindex_open(siridb, 0);
+ if (siridb->reindex != NULL && siridb->replica == NULL)
+ {
+ siridb_reindex_start(siridb->reindex->timer);
+ }
+ }
+
+ timeit_start(&siridb->start_time);
+
+ uv_mutex_lock(&siri.siridb_mutex);
+
+ /* append SiriDB to siridb_list (reference counter is already 1) */
+ if (llist_append(siri.siridb_list, siridb))
+ {
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ uv_mutex_unlock(&siri.siridb_mutex);
+
+ /* start groups update thread */
+ siridb_groups_start(siridb);
+
+ /* start tasks */
+ siridb_tasks_init(&siridb->tasks);
+
+ /* init tee if configured */
+ if (siridb_tee_is_configured(siridb->tee))
+ {
+ siridb_tee_connect(siridb->tee);
+ }
+
+ log_info("Finished loading database: '%s'", siridb->dbname);
+
+ return siridb;
+}
+
+/*
+ * Read SiriDB from unpacker. (reference counter is initially set to 1)
+ *
+ * Returns 0 if successful or a value greater than 0 if the file needs to be
+ * saved. In case of an error -1 will be returned.
+ *
+ * (a SIGNAL can be set in case of an error)
+ */
+static int siridb__from_unpacker(
+ qp_unpacker_t * unpacker,
+ siridb_t ** siridb,
+ const char * dbpath,
+ char * err_msg)
+{
+ *siridb = NULL;
+ qp_obj_t qp_obj;
+ qp_obj_t qp_schema;
+
+ if (!qp_is_array(qp_next(unpacker, NULL)) ||
+ qp_next(unpacker, &qp_schema) != QP_INT64)
+ {
+ sprintf(err_msg, "Corrupted database file.");
+ return -1;
+ }
+
+ /* check schema */
+ if ( qp_schema.via.int64 == 1 ||
+ qp_schema.via.int64 == 2 ||
+ qp_schema.via.int64 == 3 ||
+ qp_schema.via.int64 == 4 ||
+ qp_schema.via.int64 == 5)
+ {
+ log_info(
+ "Found an old database schema (v%d), "
+ "migrating to schema v%d...",
+ qp_schema.via.int64,
+ SIRIDB_SCHEMA);
+ }
+ else if (qp_schema.via.int64 != SIRIDB_SCHEMA)
+ {
+ sprintf(err_msg, "Unsupported schema found: %" PRId64,
+ qp_schema.via.int64);
+ return -1;
+ }
+
+ /* create a new SiriDB structure */
+ *siridb = siridb__new();
+ if (*siridb == NULL)
+ {
+ sprintf(err_msg, "Cannot create SiriDB instance.");
+ return -1;
+ }
+
+ /* set dbpath */
+ if (((*siridb)->dbpath = strdup(dbpath)) == NULL)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot set dbpath.")
+ }
+
+ /* read uuid */
+ if (qp_next(unpacker, &qp_obj) != QP_RAW || qp_obj.len != 16)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read uuid.")
+ }
+
+ /* copy uuid */
+ memcpy(&(*siridb)->uuid, qp_obj.via.raw, qp_obj.len);
+
+ /* read database name */
+ if (qp_next(unpacker, &qp_obj) != QP_RAW ||
+ qp_obj.len >= SIRIDB_MAX_DBNAME_LEN)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read database name.")
+ }
+
+ /* alloc mem for database name */
+ (*siridb)->dbname = strndup((const char *) qp_obj.via.raw, qp_obj.len);
+ if ((*siridb)->dbname == NULL)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot allocate database name.")
+ }
+
+ /* read time precision */
+ if (qp_next(unpacker, &qp_obj) != QP_INT64 ||
+ qp_obj.via.int64 < SIRIDB_TIME_SECONDS ||
+ qp_obj.via.int64 > SIRIDB_TIME_NANOSECONDS)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read time-precision.")
+ }
+
+ /* bind time precision to SiriDB */
+ (*siridb)->time =
+ siridb_time_new((siridb_timep_t) qp_obj.via.int64);
+ if ((*siridb)->time == NULL)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot create time instance.")
+ }
+
+ /* read buffer size, same buffer_size requirements are used in request.c */
+ if ( qp_next(unpacker, &qp_obj) != QP_INT64 ||
+ !siridb_buffer_is_valid_size(qp_obj.via.int64))
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read buffer size.")
+ }
+
+ /* bind buffer size to SiriDB */
+ (*siridb)->buffer->size = (size_t) qp_obj.via.int64;
+
+ /* read number duration */
+ if (qp_next(unpacker, &qp_obj) != QP_INT64)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read number duration.")
+ }
+
+ /* bind number duration to SiriDB */
+ (*siridb)->duration_num = (uint64_t) qp_obj.via.int64;
+
+ /* calculate 'shard_mask_num' based on number duration */
+ (*siridb)->shard_mask_num =
+ (uint16_t) sqrt((double) siridb_time_in_seconds(
+ *siridb, (*siridb)->duration_num)) / 24;
+
+ /* read log duration */
+ if (qp_next(unpacker, &qp_obj) != QP_INT64)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read log duration.")
+ }
+
+ /* bind log duration to SiriDB */
+ (*siridb)->duration_log = (uint64_t) qp_obj.via.int64;
+
+ /* calculate 'shard_mask_log' based on log duration */
+ (*siridb)->shard_mask_log =
+ (uint16_t) sqrt((double) siridb_time_in_seconds(
+ *siridb, (*siridb)->duration_log)) / 24;
+
+ log_debug("Set number duration mask to %d", (*siridb)->shard_mask_num);
+ log_debug("Set log duration mask to %d", (*siridb)->shard_mask_log);
+
+ /* read timezone */
+ if (qp_next(unpacker, &qp_obj) != QP_RAW)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read timezone.")
+ }
+
+ /* bind timezone to SiriDB */
+ char * tzname = strndup((const char *) qp_obj.via.raw, qp_obj.len);
+
+ if (tzname == NULL)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot allocate timezone name.")
+ }
+
+ if (((*siridb)->tz = iso8601_tz(tzname)) < 0)
+ {
+ log_critical("Unknown timezone found: '%s'.", tzname);
+ free(tzname);
+ READ_DB_EXIT_WITH_ERROR("Cannot read timezone.")
+ }
+ free(tzname);
+
+ /* read drop threshold */
+ if (qp_next(unpacker, &qp_obj) != QP_DOUBLE ||
+ qp_obj.via.real < 0.0 || qp_obj.via.real > 1.0)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read drop threshold.")
+ }
+ (*siridb)->drop_threshold = qp_obj.via.real;
+
+ if (qp_schema.via.int64 == 1)
+ {
+ (*siridb)->select_points_limit = DEF_SELECT_POINTS_LIMIT;
+ (*siridb)->list_limit = DEF_LIST_LIMIT;
+ }
+ else
+ {
+ /* read select points limit */
+ if (qp_next(unpacker, &qp_obj) != QP_INT64 || qp_obj.via.int64 < 1)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read select points limit.")
+ }
+ (*siridb)->select_points_limit = qp_obj.via.int64;
+
+ /* read list limit */
+ if (qp_next(unpacker, &qp_obj) != QP_INT64 || qp_obj.via.int64 < 1)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read list limit.")
+ }
+ (*siridb)->list_limit = qp_obj.via.int64;
+ }
+
+ /* for older schemas we keep the default tee_pipe_name=NULL */
+ if (qp_schema.via.int64 >= 5)
+ {
+ qp_next(unpacker, &qp_obj);
+
+ if (qp_obj.tp == QP_RAW)
+ {
+ (*siridb)->tee->pipe_name_ = strndup(
+ (char *) qp_obj.via.raw,
+ qp_obj.len);
+
+ if (!(*siridb)->tee->pipe_name_)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot allocate tee pipe name.")
+ }
+ }
+ else if (qp_obj.tp != QP_NULL)
+ {
+ READ_DB_EXIT_WITH_ERROR("Cannot read tee pipe name.")
+ }
+ }
+ if (qp_schema.via.int64 >= 6)
+ {
+ /* read select points limit */
+ if (qp_next(unpacker, &qp_obj) != QP_INT64 || qp_obj.via.int64 < 0)
+ {
+ READ_DB_EXIT_WITH_ERROR(
+ "Cannot read shard (log) expiration time.")
+ }
+ (*siridb)->expiration_log = qp_obj.via.int64;
+
+ /* read list limit */
+ if (qp_next(unpacker, &qp_obj) != QP_INT64 || qp_obj.via.int64 < 0)
+ {
+ READ_DB_EXIT_WITH_ERROR(
+ "Cannot read shard (number) expiration time.")
+ }
+ (*siridb)->expiration_num = qp_obj.via.int64;
+ }
+ if ((*siridb)->tee->pipe_name_ == NULL)
+ {
+ log_debug(
+ "No tee pipe name configured for database: %s",
+ (*siridb)->dbname);
+ }
+ else
+ {
+ log_debug(
+ "Using tee pipe name '%s' for database: '%s'",
+ (*siridb)->tee->pipe_name_,
+ (*siridb)->dbname);
+ }
+
+ return (qp_schema.via.int64 == SIRIDB_SCHEMA) ? 0 : qp_schema.via.int64;
+}
+
+/*
+ * Get a siridb object by name.
+ */
+siridb_t * siridb_get(llist_t * siridb_list, const char * dbname)
+{
+ llist_node_t * node = siridb_list->first;
+ siridb_t * siridb;
+
+ while (node != NULL)
+ {
+ siridb = (siridb_t *) node->data;
+ if (strcmp(siridb->dbname, dbname) == 0)
+ {
+ return siridb;
+ }
+ node = node->next;
+ }
+
+ return NULL;
+}
+
+/*
+ * Get a siridb object by name.
+ */
+siridb_t * siridb_getn(llist_t * siridb_list, const char * dbname, size_t n)
+{
+ llist_node_t * node = siridb_list->first;
+ siridb_t * siridb;
+
+ while (node != NULL)
+ {
+ siridb = (siridb_t *) node->data;
+ if (n == strlen(siridb->dbname) &&
+ strncmp(siridb->dbname, dbname, n) == 0)
+ {
+ return siridb;
+ }
+ node = node->next;
+ }
+
+ return NULL;
+}
+
+/*
+ * Get a siridb object by qpack name.
+ */
+siridb_t * siridb_get_by_qp(llist_t * siridb_list, qp_obj_t * qp_dbname)
+{
+ assert (qp_dbname->tp == QP_RAW);
+
+ llist_node_t * node = siridb_list->first;
+ siridb_t * siridb;
+
+ while (node != NULL)
+ {
+ siridb = (siridb_t *) node->data;
+ if (qp_dbname->len == strlen(siridb->dbname) &&
+ strncmp(
+ siridb->dbname,
+ (const char *) qp_dbname->via.raw,
+ qp_dbname->len) == 0)
+ {
+ return siridb;
+ }
+ node = node->next;
+ }
+
+ return NULL;
+}
+
+
+/*
+ * Sometimes we need a callback function and cannot use a macro expansion.
+ */
+int siridb_decref_cb(siridb_t * siridb, void * args __attribute__((unused)))
+{
+ siridb_decref(siridb);
+ return 0;
+}
+
+/*
+ * Typedef: sirinet_clserver_get_file
+ *
+ * Returns the length of the content for a file and set buffer with the file
+ * content. Note that malloc is used to allocate memory for the buffer.
+ *
+ * In case of an error -1 is returned and buffer will be set to NULL.
+ */
+ssize_t siridb_get_file(char ** buffer, siridb_t * siridb)
+{
+ /* get servers file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, "database.dat")
+
+ return xpath_get_content(buffer, fn);
+}
+
+/*
+ * Returns the number of open files by the given database.
+ * (includes both the database and buffer path)
+ */
+int siridb_open_files(siridb_t * siridb)
+{
+ siridb_buffer_t * buffer = siridb->buffer;
+ return procinfo_open_files(
+ siridb->dbpath,
+ (buffer->fp == NULL) ? -1 : buffer->fd);
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ */
+int siridb_save(siridb_t * siridb)
+{
+ char buffer[XPATH_MAX];
+ buffer[XPATH_MAX-1] = '\0';
+ snprintf(buffer,
+ XPATH_MAX-1,
+ "%sdatabase.dat",
+ siridb->dbpath);
+
+ qp_fpacker_t * fpacker;
+
+ if ((fpacker = qp_open(buffer, "w")) == NULL)
+ {
+ return -1;
+ }
+
+ return (qp_fadd_type(fpacker, QP_ARRAY_OPEN) ||
+ qp_fadd_int64(fpacker, SIRIDB_SCHEMA) ||
+ qp_fadd_raw(fpacker, (const unsigned char *) siridb->uuid, 16) ||
+ qp_fadd_string(fpacker, siridb->dbname) ||
+ qp_fadd_int64(fpacker, siridb->time->precision) ||
+ qp_fadd_int64(fpacker, siridb->buffer->size) ||
+ qp_fadd_int64(fpacker, siridb->duration_num) ||
+ qp_fadd_int64(fpacker, siridb->duration_log) ||
+ qp_fadd_string(fpacker, iso8601_tzname(siridb->tz)) ||
+ qp_fadd_double(fpacker, siridb->drop_threshold) ||
+ qp_fadd_int64(fpacker, siridb->select_points_limit) ||
+ qp_fadd_int64(fpacker, siridb->list_limit) ||
+ (siridb->tee->pipe_name_ == NULL
+ ? qp_fadd_type(fpacker, QP_NULL)
+ : qp_fadd_string(fpacker, siridb->tee->pipe_name_)) ||
+ qp_fadd_int64(fpacker, siridb->expiration_log) ||
+ qp_fadd_int64(fpacker, siridb->expiration_num) ||
+ qp_fadd_type(fpacker, QP_ARRAY_CLOSE) ||
+ qp_close(fpacker));
+}
+
+
+/*
+ * Destroy SiriDB object.
+ *
+ * Never call this function but rather call siridb_decref.
+ */
+void siridb__free(siridb_t * siridb)
+{
+ /* first we should close the buffer and all other open files */
+ if (siridb->buffer != NULL)
+ {
+ siridb_buffer_close(siridb->buffer);
+ }
+
+ if (siridb->dropped_fp != NULL)
+ {
+ fclose(siridb->dropped_fp);
+ }
+
+ if (siridb->store != NULL)
+ {
+ qp_close(siridb->store);
+ }
+
+ /* free users */
+ if (siridb->users != NULL)
+ {
+ siridb_users_free(siridb->users);
+ }
+
+ /* we do not need to free server and replica since they exist in
+ * this list and therefore will be freed.
+ */
+ if (siridb->servers != NULL)
+ {
+ siridb_servers_free(siridb->servers);
+ }
+
+ /*
+ * Destroy replicate before fifo but after servers so open promises are
+ * closed which might depend on siridb->replicate
+ *
+ * siridb->replicate must be closed, see 'SIRI_set_closing_state'
+ */
+ if (siridb->replicate != NULL)
+ {
+ siridb_replicate_free(&siridb->replicate);
+ }
+
+ if (siridb->reindex != NULL)
+ {
+ siridb_reindex_free(&siridb->reindex);
+ }
+
+ /* free fifo (in case we have a replica) */
+ if (siridb->fifo != NULL)
+ {
+ siridb_fifo_free(siridb->fifo);
+ }
+
+ /* free pools */
+ if (siridb->pools != NULL)
+ {
+ siridb_pools_free(siridb->pools);
+ }
+
+ /* free imap (series) */
+ if (siridb->series_map != NULL)
+ {
+ imap_free(siridb->series_map, NULL);
+ }
+
+ /* free c-tree lookup and series */
+ if (siridb->series != NULL)
+ {
+ ct_free(siridb->series, (ct_free_cb) &siridb__series_decref);
+ }
+
+ /* free shards using imap walk an free the imap */
+ if (siridb->shards != NULL)
+ {
+ imap_free(siridb->shards, (imap_free_cb) &siridb_shards_destroy_cb);
+ }
+
+ if (siridb->groups != NULL)
+ {
+ uv_thread_join(&siridb->groups->thread);
+ siridb_groups_decref(siridb->groups);
+ }
+
+ if (siridb->tags != NULL)
+ {
+ siridb_tags_decref(siridb->tags);
+ }
+
+ if (siridb->buffer != NULL)
+ {
+ siridb_buffer_free(siridb->buffer);
+ }
+
+ if (siridb->tee != NULL)
+ {
+ siridb_tee_free(siridb->tee);
+ }
+
+ /* unlock the database in case no siri_err occurred */
+ if (!siri_err)
+ {
+ lock_t lock_rc = lock_unlock(siridb->dbpath);
+ if (lock_rc != LOCK_REMOVED)
+ {
+ log_error("%s", lock_str(lock_rc));
+ }
+ }
+
+ uv_mutex_destroy(&siridb->series_mutex);
+ uv_mutex_destroy(&siridb->shards_mutex);
+ uv_mutex_destroy(&siridb->values_mutex);
+
+ if (siridb->flags & SIRIDB_FLAG_DROPPED)
+ {
+ xpath_rmdir(siridb->dbpath);
+ }
+
+ free(siridb->dbpath);
+ free(siridb->dbname);
+ free(siridb->time);
+ free(siridb);
+}
+
+void siridb_drop(siridb_t * siridb)
+{
+ if (siridb->flags & SIRIDB_FLAG_DROPPED)
+ {
+ return;
+ }
+
+ log_warning("dropping database '%s'", siridb->dbname);
+
+ siridb->flags |= SIRIDB_FLAG_DROPPED;
+
+ uv_mutex_lock(&siri.siridb_mutex);
+
+ llist_remove(siri.siridb_list, NULL, siridb);
+
+ uv_mutex_unlock(&siri.siridb_mutex);
+
+ if (siridb->replicate != NULL)
+ {
+ siridb_replicate_close(siridb->replicate);
+ }
+
+ if (siridb->reindex != NULL && siridb->reindex->timer != NULL)
+ {
+ siridb_reindex_close(siridb->reindex);
+ }
+
+ if (siridb->groups != NULL)
+ {
+ siridb_groups_destroy(siridb->groups);
+ }
+
+ siridb_decref(siridb);
+}
+
+void siridb_update_shard_expiration(siridb_t * siridb)
+{
+ struct timespec now;
+ clock_gettime(CLOCK_REALTIME, &now);
+
+ uv_mutex_lock(&siridb->values_mutex);
+
+ siridb->exp_at_num = siridb->expiration_num
+ ? siridb_time_now(siridb, now) - siridb->expiration_num
+ : 0;
+
+ siridb->exp_at_log = siridb->expiration_log
+ ? siridb_time_now(siridb, now) - siridb->expiration_log
+ : 0;
+
+ uv_mutex_unlock(&siridb->values_mutex);
+}
+
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+static siridb_t * siridb__new(void)
+{
+ siridb_t * siridb = malloc(sizeof(siridb_t));
+ if (siridb == NULL)
+ {
+ goto fail0;
+ }
+
+ siridb->dbname = NULL;
+ siridb->dbpath = NULL;
+ siridb->ref = 1;
+ siridb->insert_tasks = 0;
+ siridb->flags = 0;
+ siridb->time = NULL;
+ siridb->users = NULL;
+ siridb->servers = NULL;
+ siridb->pools = NULL;
+ siridb->max_series_id = 0;
+ siridb->received_points = 0;
+ siridb->selected_points = 0;
+ siridb->drop_threshold = DEF_DROP_THRESHOLD;
+ siridb->select_points_limit = DEF_SELECT_POINTS_LIMIT;
+ siridb->list_limit = DEF_LIST_LIMIT;
+ siridb->tz = -1;
+ siridb->server = NULL;
+ siridb->replica = NULL;
+ siridb->fifo = NULL;
+ siridb->replicate = NULL;
+ siridb->reindex = NULL;
+ siridb->groups = NULL;
+ siridb->groups = NULL;
+ siridb->tags = NULL;
+ siridb->store = NULL;
+ siridb->exp_at_log = 0;
+ siridb->exp_at_num = 0;
+ siridb->expiration_log = 0;
+ siridb->expiration_num = 0;
+
+ siridb->series = ct_new();
+ if (siridb->series == NULL)
+ {
+ goto fail0;
+ }
+
+ siridb->series_map = imap_new();
+ if (siridb->series_map == NULL)
+ {
+ goto fail1;
+ }
+ siridb->shards = imap_new();
+ if (siridb->shards == NULL)
+ {
+ goto fail2;
+ }
+ /* allocate a buffer */
+ siridb->buffer = siridb_buffer_new();
+ if (siridb->buffer == NULL)
+ {
+ goto fail3;
+ }
+
+ /* allocate tee */
+ siridb->tee = siridb_tee_new();
+ if (siridb->tee == NULL)
+ {
+ goto fail4;
+ }
+
+ uv_mutex_init(&siridb->series_mutex);
+ uv_mutex_init(&siridb->shards_mutex);
+ uv_mutex_init(&siridb->values_mutex);
+
+ return siridb;
+
+fail4:
+ siridb_buffer_free(siridb->buffer);
+fail3:
+ imap_free(siridb->shards, NULL);
+fail2:
+ imap_free(siridb->series_map, NULL);
+fail1:
+ ct_free(siridb->series, NULL);
+fail0:
+ free(siridb);
+ ERR_ALLOC
+ return NULL;
+}
+
+static siridb_t * siridb__from_dat(const char * dbpath)
+{
+ int rc;
+ siridb_t * siridb = NULL;
+ char err_msg[512];
+ qp_unpacker_t * unpacker;
+ char buffer[XPATH_MAX];
+ buffer[XPATH_MAX-1] = '\0';
+ snprintf(buffer,
+ XPATH_MAX-1,
+ "%sdatabase.dat",
+ dbpath);
+
+ unpacker = qp_unpacker_ff(buffer);
+ if (unpacker == NULL)
+ {
+ return NULL;
+ }
+
+ if ((rc = siridb__from_unpacker(
+ unpacker,
+ &siridb,
+ dbpath,
+ err_msg)) < 0)
+ {
+ log_error("Could not read '%s': %s", buffer, err_msg);
+ qp_unpacker_ff_free(unpacker);
+ return NULL;
+ }
+
+ qp_unpacker_ff_free(unpacker);
+
+ if (rc > 0 && siridb_save(siridb))
+ {
+ log_error("Could not write file: %s", buffer);
+ siridb_decref(siridb);
+ return NULL;
+ }
+
+ return siridb;
+}
+
+static int siridb__read_conf(siridb_t * siridb)
+{
+ int rc;
+ char buf[XPATH_MAX];
+ cfgparser_t * cfgparser;
+ cfgparser_option_t * option = NULL;
+ siridb_buffer_t * buffer = siridb->buffer;
+ buf[XPATH_MAX-1] = '\0';
+ snprintf(buf,
+ XPATH_MAX-1,
+ "%sdatabase.conf",
+ siridb->dbpath);
+
+ cfgparser = cfgparser_new();
+ if (cfgparser == NULL)
+ {
+ return -1; /* signal is raised */
+ }
+
+ rc = cfgparser_read(cfgparser, buf);
+
+ if (rc != CFGPARSER_SUCCESS)
+ {
+ log_error("Could not read '%s': %s", buf, cfgparser_errmsg(rc));
+ cfgparser_free(cfgparser);
+ return -1;
+ }
+
+ /* read buffer_path from database.conf */
+ rc = cfgparser_get_option(&option, cfgparser, "buffer", "path");
+ siridb_buffer_set_path(
+ buffer,
+ (rc == CFGPARSER_SUCCESS && option->tp == CFGPARSER_TP_STRING) ?
+ option->val->string : siridb->dbpath);
+
+ /* read buffer size from database.conf */
+ rc = cfgparser_get_option(&option, cfgparser, "buffer", "size");
+
+ if (rc == CFGPARSER_SUCCESS && option->tp == CFGPARSER_TP_INTEGER)
+ {
+ ssize_t ssize = option->val->integer;
+ if (!siridb_buffer_is_valid_size(ssize))
+ {
+ log_warning(
+ "Invalid buffer size: %" PRId64
+ " (expecting a multiple of 512 with a maximum of %" PRId64 ")",
+ ssize,
+ (int64_t) MAX_BUFFER_SZ);
+ }
+ else
+ {
+ buffer->_to_size = (buffer->size == (size_t) ssize) ?
+ 0 : (size_t) ssize;
+ }
+ }
+ else
+ {
+ FILE * fp = fopen(buf, "a");
+ if (fp != NULL)
+ {
+ if (rc == CFGPARSER_ERR_SECTION_NOT_FOUND)
+ {
+ (void) fprintf(fp, "\n[buffer]\nsize = %zu\n", buffer->size);
+ }
+ else if (rc == CFGPARSER_ERR_OPTION_NOT_FOUND)
+ {
+ (void) fprintf(fp, "\nsize = %zu\n", buffer->size);
+ }
+ (void) fclose(fp);
+ }
+ }
+ cfgparser_free(cfgparser);
+
+ return (buffer->path == NULL) ? -1 : 0;
+}
+
+static int siridb__lock(const char * dbpath, int lock_flags)
+{
+ lock_t lock_rc = lock_lock(dbpath, lock_flags);
+
+ switch (lock_rc)
+ {
+ case LOCK_IS_LOCKED_ERR:
+ case LOCK_PROCESS_NAME_ERR:
+ case LOCK_WRITE_ERR:
+ case LOCK_READ_ERR:
+ case LOCK_MEM_ALLOC_ERR:
+ log_error("%s (%s)", lock_str(lock_rc), dbpath);
+ return -1;
+ case LOCK_NEW:
+ log_info("%s (%s)", lock_str(lock_rc), dbpath);
+ break;
+ case LOCK_OVERWRITE:
+ log_warning("%s (%s)", lock_str(lock_rc), dbpath);
+ break;
+ default:
+ assert (0);
+ break;
+ }
+ return 0;
+}
--- /dev/null
+/*
+ * ffile.c - FIFO file.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <ctype.h>
+#include <logger/logger.h>
+#include <siri/db/ffile.h>
+#include <siri/err.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define FFILE_DEFAULT_SIZE 104857600 /* 100 MB */
+#define FFILE_NUMBERS 9 /* how much numbers are used to generate the file. */
+
+/*
+ * Open the fifo file. (set both the file pointer and file descriptor
+ * In case of and error, fifo->fp is set to NULL
+ */
+void siridb_ffile_open(siridb_ffile_t * ffile, const char * opentype)
+{
+ ffile->fp = fopen(ffile->fn, opentype);
+ if (ffile->fp != NULL)
+ {
+ ffile->fd = fileno(ffile->fp);
+ if (ffile->fd == -1)
+ {
+ log_critical("Error reading file descriptor: '%s'", ffile->fn);
+ fclose(ffile->fp);
+ ffile->fp = NULL;
+ }
+ }
+}
+
+/*
+ * returns NULL in case an error has occurred and a signal is set if this
+ * was a critical error.
+ */
+siridb_ffile_t * siridb_ffile_new(
+ uint64_t id,
+ const char * path,
+ sirinet_pkg_t * pkg)
+{
+ siridb_ffile_t * ffile = malloc(sizeof(siridb_ffile_t));
+
+ if (ffile == NULL)
+ {
+ ERR_ALLOC
+ return NULL;
+ }
+
+ if (asprintf(&ffile->fn, "%s%0*" PRIu64 ".fifo",
+ path,
+ FFILE_NUMBERS,
+ id) < 0)
+ {
+ ERR_ALLOC
+ free(ffile);
+ return NULL;
+ }
+
+ ffile->id = id;
+ ffile->next_size = 0;
+
+ siridb_ffile_open(ffile, "r+");
+
+ if (ffile->fp == NULL)
+ {
+ /* create a new fifo file */
+ siridb_ffile_open(ffile, "w+");
+
+ if (ffile->fp == NULL)
+ {
+ ERR_FILE
+ siridb_ffile_free(ffile);
+ return NULL;
+ }
+
+ if (pkg == NULL)
+ {
+ ffile->size = ffile->free_space = FFILE_DEFAULT_SIZE;
+ }
+ else
+ {
+ /* we need 4 extra zeros (uint32_t) at the start of a fifo file */
+ size_t size = pkg->len + sizeof(sirinet_pkg_t) + 2 * sizeof(uint32_t);
+
+ /* set free space to a value is will always fit */
+ ffile->size = ffile->free_space = (size > FFILE_DEFAULT_SIZE) ?
+ size : FFILE_DEFAULT_SIZE;
+
+ /* because we has enough free space, this should always work */
+ if (siridb_ffile_append(ffile, pkg) != FFILE_SUCCESS)
+ {
+ ERR_FILE;
+ siridb_ffile_free(ffile);
+ return NULL;
+ }
+ }
+ }
+ else
+ {
+ /* reading existing fifo file */
+
+ if (fseeko(ffile->fp, 0, SEEK_END))
+ {
+ ERR_FILE
+ siridb_ffile_free(ffile);
+ return NULL;
+ }
+
+ if ((ffile->size = ftello(ffile->fp)) >= (off_t) sizeof(uint32_t))
+ {
+ ffile->free_space = 0;
+ if ( fseeko(ffile->fp, -(long int) sizeof(uint32_t), SEEK_END) ||
+ fread( &ffile->next_size,
+ sizeof(uint32_t),
+ 1,
+ ffile->fp) != 1 ||
+ fclose(ffile->fp))
+ {
+ ERR_FILE
+ siridb_ffile_free(ffile);
+ return NULL;
+ }
+ ffile->fp = NULL;
+ }
+
+ if (!ffile->next_size)
+ {
+ log_debug("Empty fifo found, removing: '%s'", ffile->fn);
+ /*
+ * signal can be set in unlink, otherwise it's just an empty
+ * fifo and can be removed.
+ */
+ siridb_ffile_unlink(ffile);
+ ffile = NULL;
+ }
+ }
+
+ return ffile;
+}
+
+/*
+ * returns a result code, usually FFILE_ERROR is critical but signal
+ * will never be set by this function.
+ */
+siridb_ffile_result_t siridb_ffile_append(
+ siridb_ffile_t * ffile,
+ sirinet_pkg_t * pkg)
+{
+ assert (ffile->fp != NULL);
+
+ uint32_t size = pkg->len + sizeof(sirinet_pkg_t);
+
+ if (ffile->free_space < size + 2 * sizeof(uint32_t))
+ {
+ ffile->free_space = 0;
+ return FFILE_NO_FREE_SPACE;
+ }
+
+ if (!ffile->next_size)
+ {
+ ffile->next_size = size;
+ }
+ ffile->free_space -= size + sizeof(uint32_t);
+
+ if ( fseeko(ffile->fp, (off_t) ffile->free_space, SEEK_SET) ||
+ fwrite((unsigned char *) pkg, size, 1, ffile->fp) != 1 ||
+ fwrite(&size, sizeof(uint32_t), 1, ffile->fp) != 1 ||
+ fflush(ffile->fp))
+ {
+ return FFILE_ERROR;
+ }
+
+ return FFILE_SUCCESS;
+}
+
+/*
+ * returns 1 (true) if the file name is valid and 0 (false) if not
+ */
+int siridb_ffile_check_fn(const char * fn)
+{
+ int i = 0;
+ while (*fn && isdigit(*fn))
+ {
+ fn++;
+ i++;
+ }
+ return (i == FFILE_NUMBERS) ? (strcmp(fn, ".fifo") == 0) : 0;
+}
+
+/*
+ * returns a package object or NULL in case of an error.
+ *
+ * warning: be sure to check 'next_size' before calling this function.
+ */
+sirinet_pkg_t * siridb_ffile_pop(siridb_ffile_t * ffile)
+{
+ assert (ffile->next_size);
+ assert (ffile->fp != NULL);
+ if (fseeko(
+ ffile->fp,
+ -(long int) (ffile->next_size + sizeof(uint32_t)),
+ SEEK_END))
+ {
+ log_critical("Seek error in '%s'", ffile->fn);
+ return NULL;
+ }
+ sirinet_pkg_t * pkg = malloc(ffile->next_size);
+
+ if (pkg == NULL)
+ {
+ ERR_ALLOC
+ return NULL;
+ }
+
+ if (fread(pkg, ffile->next_size, 1, ffile->fp) != 1)
+ {
+ log_critical(
+ "Error while reading %" PRIu32 " bytes from '%s'",
+ ffile->next_size,
+ ffile->fn);
+ free(pkg);
+ return NULL;
+ }
+
+ if ( ffile->next_size < sizeof(sirinet_pkg_t) ||
+ pkg->len != ffile->next_size - sizeof(sirinet_pkg_t))
+ {
+ log_critical(
+ "Corrupt package in fifo: '%s' ", ffile->fn);
+ free(pkg);
+ return NULL;
+ }
+
+ return pkg;
+}
+
+/*
+ * returns 0 if successful, -1 in case of an error
+ *
+ * Warning: ffile->size might be incorrect if an error occurred
+ */
+int siridb_ffile_pop_commit(siridb_ffile_t * ffile)
+{
+ assert (ffile->next_size && ffile->fp != NULL);
+
+ ffile->size -= ffile->next_size + sizeof(uint32_t);
+
+ return (fseeko(
+ ffile->fp,
+ ffile->size - sizeof(uint32_t),
+ SEEK_SET) ||
+ fread(&ffile->next_size, sizeof(uint32_t), 1, ffile->fp) != 1 ||
+ ftruncate(ffile->fd, ffile->size)) ?
+ -1 : 0;
+}
+
+
+/*
+ * signal can be set in case of file errors
+ */
+void siridb_ffile_unlink(siridb_ffile_t * ffile)
+{
+ if (ffile->fp != NULL && fclose(ffile->fp))
+ {
+ ERR_FILE
+ }
+ if (unlink(ffile->fn))
+ {
+ ERR_FILE
+ log_critical("Cannot remove fifo file: '%s'", ffile->fn);
+ }
+ free(ffile->fn);
+ free(ffile);
+}
+
+/*
+ * signal can be set in case of file errors
+ */
+void siridb_ffile_free(siridb_ffile_t * ffile)
+{
+ if (ffile->fp != NULL && fclose(ffile->fp))
+ {
+ ERR_FILE
+ }
+ free(ffile->fn);
+ free(ffile);
+}
--- /dev/null
+/*
+ * fifo.c - First in, first out file buffer .
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <ctype.h>
+#include <limits.h>
+#include <logger/logger.h>
+#include <siri/db/fifo.h>
+#include <siri/err.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <uuid/uuid.h>
+
+static int FIFO_walk_free(siridb_ffile_t * ffile, void * args);
+static int FIFO_init(siridb_fifo_t * fifo);
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ *
+ * Make sure siridb->replica is not NULL since this function needs its UUID.
+ */
+siridb_fifo_t * siridb_fifo_new(siridb_t * siridb)
+{
+ siridb_fifo_t * fifo = malloc(sizeof(siridb_fifo_t));
+
+ if (fifo == NULL)
+ {
+ ERR_ALLOC
+ return NULL;
+ }
+
+ fifo->fifos = llist_new();
+
+ if (fifo->fifos == NULL)
+ {
+ free(fifo);
+ ERR_ALLOC
+ return NULL;
+ }
+
+ fifo->in = NULL;
+ fifo->out = NULL;
+
+ char str_uuid[37];
+ uuid_unparse_lower(siridb->replica->uuid, str_uuid);
+
+ if (asprintf(&fifo->path, "%s.%s/", siridb->dbpath, str_uuid) < 0)
+ {
+ ERR_ALLOC
+ siridb_fifo_free(fifo);
+ return NULL;
+ }
+
+ if (FIFO_init(fifo))
+ {
+ siridb_fifo_free(fifo);
+ return NULL;
+ }
+
+ fifo->max_id = (fifo->fifos->len) ?
+ (ssize_t) ((siridb_ffile_t *) fifo->fifos->last->data)->id : -1;
+
+ fifo->in = siridb_ffile_new(++fifo->max_id, fifo->path, NULL);
+ if (fifo->in == NULL)
+ {
+ ERR_FILE
+ siridb_fifo_free(fifo);
+ return NULL;
+ }
+
+ if (llist_append(fifo->fifos, fifo->in))
+ {
+ siridb_fifo_free(fifo);
+ ERR_ALLOC
+ return NULL;
+ }
+
+ /* we have at least one fifo in the list */
+ fifo->out = (siridb_ffile_t *) llist_shift(fifo->fifos);
+
+ assert (fifo->out != NULL);
+
+ if (fifo->out->fp == NULL)
+ {
+ siridb_ffile_open(fifo->out, "r+");
+ if (fifo->out->fp == NULL)
+ {
+ ERR_FILE
+ log_critical("Cannot open file: '%s'", fifo->out->fn);
+ siridb_fifo_free(fifo);
+ return NULL;
+ }
+ }
+
+ return fifo;
+}
+
+/*
+ * Returns 0 if successful or anything else if not.
+ * (signal is set in case of an error)
+ */
+int siridb_fifo_append(siridb_fifo_t * fifo, sirinet_pkg_t * pkg)
+{
+ switch(siridb_ffile_append(fifo->in, pkg))
+ {
+ case FFILE_NO_FREE_SPACE:
+ if (fifo->in != fifo->out)
+ {
+ if (fclose(fifo->in->fp))
+ {
+ ERR_FILE
+ }
+ fifo->in->fp = NULL;
+ }
+
+ fifo->in = siridb_ffile_new(++fifo->max_id, fifo->path, pkg);
+
+ if (!fifo->out->next_size)
+ {
+ /* when the out fifo has no next size, we want to use the new
+ * in fifo also as the out fifo.
+ */
+ siridb_ffile_unlink(fifo->out);
+ fifo->out = fifo->in;
+ }
+ else if (llist_append(fifo->fifos, fifo->in))
+ {
+ ERR_ALLOC;
+ }
+ break;
+ case FFILE_SUCCESS:
+ break;
+ case FFILE_ERROR:
+ ERR_FILE
+ break;
+ }
+ return siri_err;
+}
+
+/*
+ * returns a package created with malloc or NULL when an error has occurred.
+ * (signal is set in case of a malloc error, not in case of a file error)
+ *
+ * warning:
+ * be sure to check the fifo using siridb_fifo_has_data() and
+ * siridb_fifo_is_open() before calling this function.
+ */
+sirinet_pkg_t * siridb_fifo_pop(siridb_fifo_t * fifo)
+{
+ sirinet_pkg_t * pkg = siridb_ffile_pop(fifo->out);
+ if (pkg == NULL && !siri_err)
+ {
+ /*
+ * In case siri_err is not set, we can try to recover by commiting an
+ * error. We should not do this in case of malloc errors.
+ */
+ siridb_fifo_commit_err(fifo);
+ }
+ return pkg;
+}
+
+/*
+ * returns 0 if successful or another value in case of errors.
+ * (signal can be set when result is not 0)
+ */
+int siridb_fifo_commit(siridb_fifo_t * fifo)
+{
+ if (siridb_ffile_pop_commit(fifo->out))
+ {
+ log_error("Error occurred when shrinking file: '%s' ",
+ fifo->out->fn);
+ if (fifo->out != fifo->in)
+ {
+ log_warning(
+ "We try to recover from this error by removing this fifo.");
+ fifo->out->next_size = 0;
+ }
+ else
+ {
+ ERR_FILE
+ log_critical(
+ "The current fifo buffer is corrupt and we currently "
+ "cannot recover from this error. "
+ "(a reboot might remove the corrupt fifo)");
+ }
+ }
+
+ if (!fifo->out->next_size && fifo->out != fifo->in)
+ {
+ siridb_ffile_unlink(fifo->out);
+ fifo->out = (siridb_ffile_t *) llist_shift(fifo->fifos);
+
+ /* fifo->out->fp can be open in case it is equal to fifo->in */
+ if (fifo->out->fp == NULL)
+ {
+ siridb_ffile_open(fifo->out, "r+");
+ if (fifo->out->fp == NULL)
+ {
+ ERR_FILE
+ log_critical("Cannot open file: '%s'", fifo->out->fn);
+ }
+ }
+ }
+
+ assert (fifo->out != NULL);
+
+ return siri_err;
+}
+
+/*
+ * returns 0 if successful or another value in case of errors.
+ * (signal can be set when result is not 0)
+ */
+int siridb_fifo_commit_err(siridb_fifo_t * fifo)
+{
+ log_error(
+ "Handling the last package from the fifo buffer has failed, "
+ "commit the pop anyway");
+ return siridb_fifo_commit(fifo);
+}
+
+/*
+ * returns 0 if successful or a negative value in case of errors
+ */
+int siridb_fifo_close(siridb_fifo_t * fifo)
+{
+ assert (fifo->in->fp != NULL);
+ int rc = 0;
+
+ /* close the 'in' fifo */
+ rc += fclose(fifo->in->fp);
+ fifo->in->fp = NULL;
+
+ /* if 'out' is not the same as 'in', we also need to close 'out' */
+ if (fifo->out->fp != NULL)
+ {
+ rc += fclose(fifo->out->fp);
+ fifo->out->fp = NULL;
+ }
+
+ /* return 0 if successful or a negative value in case of errors */
+ return rc;
+}
+
+/*
+ * returns 0 if successful or a -1 in case of errors
+ */
+int siridb_fifo_open(siridb_fifo_t * fifo)
+{
+ assert (fifo->in->fp == NULL);
+ /* open fifo 'in' */
+ siridb_ffile_open(fifo->in, "r+");
+
+ /* if 'out' is not the same as 'in', we also need to open 'out' */
+ if (fifo->out->fp == NULL)
+ {
+ siridb_ffile_open(fifo->out, "r+");
+ }
+
+ return (fifo->in->fp != NULL && fifo->out->fp != NULL) ? 0 : -1;
+}
+
+/*
+ * destroy the fifo and close open files.
+ * (signal is set if a file close has failed)
+ */
+void siridb_fifo_free(siridb_fifo_t * fifo)
+{
+ /* we only need to free fifo->out because fido->in is either in the
+ * list or the same as fifo->out. (fifo->out is never in the list)
+ */
+ siridb_ffile_free(fifo->out);
+
+ llist_free_cb(fifo->fifos, (llist_cb) FIFO_walk_free, NULL);
+ free(fifo->path);
+ free(fifo);
+}
+
+/*
+ * returns the number of fifo files.
+ * (in case fifo is NULL, the return value will be zero)
+ */
+size_t siridb_fifo_size(siridb_fifo_t * fifo)
+{
+ return (fifo == NULL) ? 0 : fifo->fifos->len + 1;
+}
+
+
+/*
+ * returns 1 and a signal can be set if a file close has failed
+ */
+static int FIFO_walk_free(
+ siridb_ffile_t * ffile,
+ void * args __attribute__((unused)))
+{
+ siridb_ffile_free(ffile);
+ return 1;
+}
+
+/*
+ * returns 0 when successful or any other value when not.
+ * (in case of an error a signal is set too)
+ */
+static int FIFO_init(siridb_fifo_t * fifo)
+{
+ struct stat st;
+ memset(&st, 0, sizeof(struct stat));
+
+ siridb_ffile_t * ffile;
+
+ if (stat(fifo->path, &st) == -1)
+ {
+ log_warning(
+ "Fifo directory not found, creating directory '%s'.",
+ fifo->path);
+ if (mkdir(fifo->path, 0700) == -1)
+ {
+ log_critical("Cannot create directory '%s'.", fifo->path);
+ ERR_C
+ }
+ }
+ else
+ {
+ struct dirent ** fifo_list;
+ char * fn;
+ int total = scandir(fifo->path, &fifo_list, NULL, alphasort);
+ int n;
+
+ if (total < 0)
+ {
+ /* no need to free fifo_list when total < 0 */
+ log_critical("Cannot read fifo directory '%s'.", fifo->path);
+ ERR_C
+ }
+
+ for (n = 0; n < total; n++)
+ {
+ if (siridb_ffile_check_fn(fifo_list[n]->d_name))
+ {
+ if (asprintf(&fn, "%s%s", fifo->path, fifo_list[n]->d_name) < 0)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ uint64_t id = strtoull(fifo_list[n]->d_name, NULL, 10);
+ ffile = siridb_ffile_new(id, fifo->path, NULL);
+ if (ffile != NULL && llist_append(fifo->fifos, ffile))
+ {
+ ERR_ALLOC;
+ }
+ free(fn);
+ }
+ }
+ free(fifo_list[n]);
+ }
+ free(fifo_list);
+ }
+ return siri_err;
+}
+
+
--- /dev/null
+/*
+ * forward.c - Handle forwarding series while re-indexing.
+ */
+#include <qpack/qpack.h>
+#include <siri/async.h>
+#include <siri/db/forward.h>
+#include <siri/err.h>
+#include <siri/net/promises.h>
+#include <siri/net/protocol.h>
+#include <stddef.h>
+
+static void FORWARD_on_response(vec_t * promises, uv_async_t * handle);
+static void FORWARD_free(uv_handle_t * handle);
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+siridb_forward_t * siridb_forward_new(siridb_t * siridb)
+{
+ uint16_t size = siridb->pools->len;
+
+ siridb_forward_t * forward = malloc(
+ sizeof(siridb_forward_t) + size * sizeof(qp_packer_t *));
+
+ if (forward == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ uint32_t psize;
+ size_t n;
+
+ forward->free_cb = FORWARD_free;
+ forward->ref = 1; /* used as reference on (siri_async_t) handle */
+ forward->siridb = siridb;
+
+ /*
+ * we keep the packer size because the number of pools might change and
+ * at this point the pool->len is equal to when the insert was received
+ */
+ forward->size = size;
+
+ /*
+ * Allocate packers for sending data to pools. we allocate smaller
+ * sizes in case we have a lot of pools.
+ */
+ psize = QP_SUGGESTED_SIZE / ((size / 4) + 1);
+
+ for (n = 0; n < size; n++)
+ {
+ if (n == siridb->server->pool)
+ {
+ forward->packer[n] = NULL;
+ }
+ else if ((forward->packer[n] = sirinet_packer_new(psize)) != NULL)
+ {
+ /* cannot raise a signal since enough space is allocated */
+ qp_add_type(forward->packer[n], QP_MAP_OPEN);
+ }
+ else
+ {
+ return NULL; /* a signal is raised */
+ }
+ }
+ }
+ return forward;
+}
+
+/*
+ * Destroy forward.
+ */
+void siridb_forward_free(siridb_forward_t * forward)
+{
+ size_t n;
+
+ /* free packer */
+ for (n = 0; n < forward->size; n++)
+ {
+ if (forward->packer[n] != NULL)
+ {
+ qp_packer_free(forward->packer[n]);
+ }
+ }
+
+ /* free forward */
+ free(forward);
+}
+
+/*
+ * Call-back function: uv_async_cb
+ *
+ * In case of an error a SIGNAL is raised and a successful message will not
+ * be send to the client.
+ */
+void siridb_forward_points_to_pools(uv_async_t * handle)
+{
+ siridb_forward_t * forward = (siridb_forward_t *) handle->data;
+ sirinet_pkg_t * pkg;
+ sirinet_promises_t * promises = sirinet_promises_new(
+ forward->size,
+ (sirinet_promises_cb) FORWARD_on_response,
+ handle,
+ NULL);
+ int pool_count = 0;
+ uint16_t n;
+
+ if (promises == NULL)
+ {
+ return; /* signal is raised */
+ }
+
+ for (n = 0; n < forward->size; n++)
+ {
+ if ( forward->packer[n] == NULL ||
+ forward->packer[n]->len == sizeof(sirinet_pkg_t) + 1)
+ {
+ /*
+ * skip empty packer and NULL.
+ * (empty packer has only sizeof(sirinet_pkg_t) + QP_MAP_OPEN)
+ */
+ continue;
+ }
+ pkg = sirinet_packer2pkg(
+ forward->packer[n],
+ 0,
+ BPROTO_INSERT_TESTED_POOL);
+
+ /* the packer is destroyed, set to NULL */
+ forward->packer[n] = NULL;
+
+ if (siridb_pool_send_pkg(
+ forward->siridb->pools->pool + n,
+ pkg,
+ 0,
+ (sirinet_promise_cb) sirinet_promises_on_response,
+ promises,
+ 0))
+ {
+ log_critical("One pool is unreachable while re-indexing!");
+ free(pkg);
+ /*
+ * TODO: we can add the package to some retry queue for this pool.
+ */
+ }
+ else
+ {
+ pool_count++;
+ }
+ }
+
+ /* pool_count is always smaller than the initial promises->size */
+ promises->promises->size = pool_count;
+
+ SIRINET_PROMISES_CHECK(promises)
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * This function can raise a SIGNAL.
+ */
+static void FORWARD_on_response(vec_t * promises, uv_async_t * handle)
+{
+ if (promises != NULL)
+ {
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ siridb_forward_t * forward = (siridb_forward_t *) handle->data;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+ if (promise == NULL)
+ {
+ log_critical("Critical error occurred on '%s'",
+ forward->siridb->server->name);
+ continue;
+ }
+ pkg = promise->data;
+
+ if (pkg == NULL || pkg->tp != BPROTO_ACK_INSERT)
+ {
+ log_critical(
+ "Error occurred while sending points to at least '%s'",
+ promise->server->name);
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+ }
+
+ uv_close((uv_handle_t *) handle, siri_async_close);
+}
+
+/*
+ * Used as uv_close_cb.
+ */
+static void FORWARD_free(uv_handle_t * handle)
+{
+ siridb_forward_t * forward = (siridb_forward_t *) handle->data;
+
+ /* free forward */
+ siridb_forward_free(forward);
+
+ /* free handle */
+ free((uv_async_t *) handle);
+
+}
--- /dev/null
+/*
+ * group.c - Group (saved regular expressions).
+ */
+#include <assert.h>
+#include <siri/db/db.h>
+#include <siri/db/group.h>
+#include <siri/db/re.h>
+#include <siri/db/series.h>
+#include <siri/err.h>
+#include <siri/grammar/grammar.h>
+#include <vec/vec.h>
+#include <stdlib.h>
+#include <xstr/xstr.h>
+
+#define SIRIDB_MIN_GROUP_LEN 1
+#define SIRIDB_MAX_GROUP_LEN 255
+
+/*
+ * Returns a group or NULL in case of an error. When this error is critical,
+ * a SIGNAL is raised but in any case err_msg is set with an appropriate
+ * error message.
+ */
+siridb_group_t * siridb_group_new(
+ const char * source,
+ size_t source_len,
+ char * err_msg)
+{
+ siridb_group_t * group = malloc(sizeof(siridb_group_t));
+ if (group == NULL)
+ {
+ ERR_ALLOC
+ sprintf(err_msg, "Memory allocation error.");
+ }
+ else
+ {
+ group->ref = 1;
+ group->n = 0;
+ group->flags = GROUP_FLAG_INIT;
+ group->name = NULL;
+ group->source = strndup(source, source_len);
+ group->series = vec_new(VEC_DEFAULT_SIZE);
+ group->regex = NULL;
+ group->match_data = NULL;
+
+ if ( group->source == NULL ||
+ group->series == NULL)
+ {
+ ERR_ALLOC
+ sprintf(err_msg, "Memory allocation error.");
+ siridb__group_free(group);
+ group = NULL;
+ }
+ else if (siridb_re_compile(
+ &group->regex,
+ &group->match_data,
+ source,
+ source_len,
+ err_msg))
+ {
+ /* not critical, err_msg is set */
+ siridb__group_free(group);
+ group = NULL;
+ }
+ }
+ return group;
+}
+
+/*
+ * Returns 0 when successful or a positive value in case the name is not valid.
+ * A negative value is returned and a signal is raised in case a critical error
+ * has occurred.
+ *
+ * (err_msg is set in case of all errors)
+ */
+int siridb_group_set_name(
+ siridb_t * siridb,
+ siridb_group_t * group,
+ const char * name,
+ char * err_msg)
+{
+ if (strlen(name) < SIRIDB_MIN_GROUP_LEN)
+ {
+ sprintf(err_msg, "Group name should be at least %d characters.",
+ SIRIDB_MIN_GROUP_LEN);
+ return 1;
+ }
+
+ if (strlen(name) > SIRIDB_MAX_GROUP_LEN)
+ {
+ sprintf(err_msg, "Group name should be at most %d characters.",
+ SIRIDB_MAX_GROUP_LEN);
+ return 1;
+ }
+
+ if (ct_get(siridb->groups->groups, name) != NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Group '%s' already exists.",
+ name);
+ return 1;
+ }
+
+ if (siridb->tags && ct_get(siridb->tags->tags, name) != NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Tag '%s' already exists.",
+ name);
+ return 1;
+ }
+
+ if (group->name != NULL)
+ {
+ int rc;
+
+ /* group already exists */
+ uv_mutex_lock(&siridb->groups->mutex);
+
+ rc = ( ct_pop(siridb->groups->groups, group->name) == NULL ||
+ ct_add(siridb->groups->groups, name, group));
+
+ uv_mutex_unlock(&siridb->groups->mutex);
+
+ if (rc)
+ {
+ ERR_C
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Critical error while replacing group name '%s' with '%s' "
+ "in tree.",
+ group->name,
+ name);
+ return -1;
+ }
+ }
+
+ free(group->name);
+ group->name = strdup(name);
+
+ if (group->name == NULL)
+ {
+ ERR_ALLOC
+ sprintf(err_msg, "Memory allocation error.");
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Can be used as a callback, in other cases go for the macro.
+ */
+void siridb__group_decref(siridb_group_t * group)
+{
+ siridb_group_decref(group);
+}
+
+/*
+ * Remove dropped series from the group and shrink memory usage the the
+ * series list.
+ *
+ * (Group thread)
+ */
+void siridb_group_cleanup(siridb_group_t * group)
+{
+ size_t dropped = 0;
+ siridb_series_t * series;
+ size_t i;
+
+ for (i = 0; i < group->series->len; i++)
+ {
+ series = (siridb_series_t *) group->series->data[i];
+
+ if (series->flags & SIRIDB_SERIES_IS_DROPPED)
+ {
+ siridb_series_decref(series);
+ dropped++;
+ }
+ else if (dropped)
+ {
+ group->series->data[i - dropped] = series;
+ }
+ }
+
+ group->series->len -= dropped;
+
+ vec_compact(&group->series);
+}
+
+/*
+ * Group thread.
+ */
+int siridb_group_test_series(siridb_group_t * group, siridb_series_t * series)
+{
+ /* skip if group has flags set. (DROPPED or INIT) */
+ int rc = (group->flags) ? -1 : pcre2_match(
+ group->regex,
+ (PCRE2_SPTR8) series->name,
+ series->name_len,
+ 0, /* start looking at this point */
+ 0, /* OPTIONS */
+ group->match_data,
+ NULL); /* length of sub_str_vec */
+
+ if (rc >= 0)
+ {
+ if (vec_append_safe(&group->series, series))
+ {
+ log_critical(
+ "Cannot append series '%s' to group '%s'",
+ series->name,
+ group->name);
+ rc = -1;
+ }
+ else
+ {
+ siridb_series_incref(series);
+ rc = 0;
+ }
+ }
+
+ return rc;
+}
+
+/*
+ * Returns 0 when successful or -1 in case or an error.
+ *
+ * Signal might be raised in case of a memory error;
+ * (err_msg is always set in case of an error)
+ */
+int siridb_group_update_expression(
+ siridb_groups_t * groups,
+ siridb_group_t * group,
+ const char * source,
+ size_t source_len,
+ char * err_msg)
+{
+ char * new_source = strndup(source, source_len);
+ pcre2_code * new_regex;
+ pcre2_match_data * new_regex_match_data;
+ siridb_series_t * series;
+ size_t i;
+
+ if (new_source == NULL)
+ {
+ ERR_ALLOC
+ sprintf(err_msg, "Memory allocation error.");
+ return -1;
+ }
+
+ if (siridb_re_compile(
+ &new_regex,
+ &new_regex_match_data,
+ source,
+ source_len,
+ err_msg))
+ {
+ free(new_source);
+ return -1; /* err_msg is set */
+ }
+
+ uv_mutex_lock(&groups->mutex);
+
+ /* replace group expression */
+ free(group->source);
+ pcre2_code_free(group->regex);
+ pcre2_match_data_free(group->match_data);
+
+ group->source = new_source;
+ group->regex = new_regex;
+ group->match_data = new_regex_match_data;
+
+ for (i = 0; i < group->series->len; i++)
+ {
+ series = (siridb_series_t *) group->series->data[i];
+ siridb_series_decref(series);
+ }
+
+ group->series->len = 0;
+
+ vec_compact(&group->series);
+
+ if (~group->flags & GROUP_FLAG_INIT)
+ {
+ group->flags |= GROUP_FLAG_INIT;
+
+ if (vec_append_safe(&groups->ngroups, group))
+ {
+ /* we log critical since allocation errors are critical, this does
+ * however not influence the running SiriDB in is not critical in that
+ * perspective.
+ */
+ log_critical("Cannot add group to list for initialization.");
+ }
+ else
+ {
+ siridb_group_incref(group);
+ }
+ }
+
+ uv_mutex_unlock(&groups->mutex);
+
+ return 0;
+}
+
+int siridb_group_cexpr_cb(siridb_group_t * group, cexpr_condition_t * cond)
+{
+ switch (cond->prop)
+ {
+ case CLERI_GID_K_NAME:
+ return cexpr_str_cmp(cond->operator, group->name, cond->str);
+ case CLERI_GID_K_SERIES:
+ return cexpr_int_cmp(cond->operator, group->n, cond->int64);
+ case CLERI_GID_K_EXPRESSION:
+ return cexpr_str_cmp(cond->operator, group->source, cond->str);
+ }
+
+ log_critical("Unknown group property received: %d", cond->prop);
+ assert (0);
+ return -1;
+}
+
+/*
+ * This function can raise a SIGNAL. In this case the packer is not filled
+ * with the correct values.
+ */
+void siridb_group_prop(siridb_group_t * group, qp_packer_t * packer, int prop)
+{
+ switch (prop)
+ {
+ case CLERI_GID_K_NAME:
+ qp_add_string(packer, group->name);
+ break;
+ case CLERI_GID_K_SERIES:
+ qp_add_int64(packer, (int64_t) group->n);
+ break;
+ case CLERI_GID_K_EXPRESSION:
+ qp_add_string(packer, group->source);
+ }
+}
+
+/*
+ * Returns true when the given property (CLERI keyword) needs a remote query
+ */
+int siridb_group_is_remote_prop(uint32_t prop)
+{
+ return (prop == CLERI_GID_K_SERIES) ? 1 : 0;
+}
+
+/*
+ * NEVER call this function but rather call siridb_group_decref instead.
+ *
+ * Destroy a group object. Parsing NULL is not allowed.
+ */
+void siridb__group_free(siridb_group_t * group)
+{
+ size_t i;
+
+ free(group->name);
+ free(group->source);
+
+ if (group->series != NULL)
+ {
+ siridb_series_t * series;
+ for (i = 0; i < group->series->len; i++)
+ {
+ series = (siridb_series_t *) group->series->data[i];
+ siridb_series_decref(series);
+ }
+ vec_free(group->series);
+ }
+
+ pcre2_code_free(group->regex);
+ pcre2_match_data_free(group->match_data);
+ free(group);
+}
--- /dev/null
+/*
+ * groups.c - Groups (saved regular expressions).
+ *
+ * Info groups->mutex:
+ *
+ * Main thread:
+ * groups->groups : read (no lock) write (lock)
+ * groups->nseries : read (lock) write (lock)
+ * groups->ngroups : read (lock) write (lock)
+ * group->series : read (lock) write (not allowed)
+
+ * Other threads:
+ * groups->groups : read (lock) write (not allowed)
+ * groups->nseries : read (lock) write (lock)
+ * groups->ngroups : read (lock) write (lock)
+ *
+ * Group thread:
+ * group->series : read (no lock) write (lock)
+ *
+ * Note: One exception to 'not allowed' are the free functions
+ * since they only run when no other references to the object exist.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/db.h>
+#include <siri/db/group.h>
+#include <siri/db/groups.h>
+#include <siri/db/misc.h>
+#include <siri/db/series.h>
+#include <siri/err.h>
+#include <siri/net/protocol.h>
+#include <siri/siri.h>
+#include <vec/vec.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <xpath/xpath.h>
+
+#define SIRIDB_GROUPS_SCHEMA 1
+#define SIRIDB_GROUPS_FN "groups.dat"
+#define GROUPS_LOOP_SLEEP 2 /* 2 seconds */
+#define GROUPS_LOOP_DEEP 15 /* x times -> 30 seconds. used when re-indexing */
+#define GROUPS_RE_BATCH_SZ 1000
+#define CALC_BATCH_SIZE(sz) GROUPS_RE_BATCH_SZ /((sz / 5 ) + 1) + 1;
+
+static int GROUPS_load(siridb_t * siridb);
+static int GROUPS_pkg(siridb_group_t * group, qp_packer_t * packer);
+static int GROUPS_nseries(siridb_group_t * group, void * data);
+static void GROUPS_loop(void * arg);
+static int GROUPS_write(siridb_group_t * group, qp_fpacker_t * fpacker);
+static void GROUPS_init_groups(siridb_t * siridb);
+static void GROUPS_init_series(siridb_t * siridb);
+static int GROUPS_2vec(siridb_group_t * group, vec_t * groups_list);
+static void GROUPS_cleanup(siridb_groups_t * groups);
+/*
+ * In case of an error the return value is NULL and a SIGNAL is raised.
+ */
+int siridb_groups_init(siridb_t * siridb)
+{
+ log_info("Loading groups");
+
+ siridb->groups = malloc(sizeof(siridb_groups_t));
+ if (siridb->groups == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ siridb->groups->ref = 1;
+ siridb->groups->fn = NULL;
+ siridb->groups->groups = ct_new();
+ siridb->groups->nseries = vec_new(VEC_DEFAULT_SIZE);
+ siridb->groups->ngroups = vec_new(VEC_DEFAULT_SIZE);
+
+ uv_mutex_init(&siridb->groups->mutex);
+
+ if ( !siridb->groups->groups ||
+ !siridb->groups->nseries ||
+ !siridb->groups->ngroups)
+ {
+ ERR_ALLOC
+ siridb__groups_free(siridb->groups);
+ siridb->groups = NULL;
+ return -1;
+ }
+
+ if (asprintf(
+ &siridb->groups->fn,
+ "%s%s",
+ siridb->dbpath,
+ SIRIDB_GROUPS_FN) < 0 || GROUPS_load(siridb))
+ {
+ ERR_ALLOC
+ siridb__groups_free(siridb->groups);
+ siridb->groups = NULL;
+ return -1;
+ }
+
+ siridb->groups->status = GROUPS_RUNNING;
+ siridb->groups->flags = 0;
+ return 0;
+}
+
+/*
+ * Start group thread.
+ */
+void siridb_groups_start(siridb_t * siridb)
+{
+ uv_thread_create(&siridb->groups->thread, GROUPS_loop, siridb);
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ */
+int siridb_groups_add_series(
+ siridb_groups_t * groups,
+ siridb_series_t * series)
+{
+ int rc = 0;
+
+ uv_mutex_lock(&groups->mutex);
+
+ if (vec_append_safe(&groups->nseries, series) == 0)
+ {
+ siridb_series_incref(series);
+ }
+ else
+ {
+ log_critical("Error while initializing series '%s' for groups",
+ series->name);
+ rc = -1;
+ }
+
+ uv_mutex_unlock(&groups->mutex);
+
+ return rc;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * (a signal might be raised)
+ */
+int siridb_groups_add_group(
+ siridb_t * siridb,
+ const char * name,
+ const char * source,
+ size_t source_len,
+ char * err_msg)
+{
+ int rc;
+
+ siridb_group_t * group = siridb_group_new(
+ source,
+ source_len,
+ err_msg);
+
+ if (group == NULL)
+ {
+ return -1; /* err_msg is set and a SIGNAL is possibly raised */
+ }
+
+ if (siridb_group_set_name(siridb, group, name, err_msg))
+ {
+ siridb_group_decref(group);
+ return -1; /* err_msg is set and a SIGNAL is possibly raised */
+ }
+
+ uv_mutex_lock(&siridb->groups->mutex);
+
+ rc = ct_add(siridb->groups->groups, name, group);
+
+ switch (rc)
+ {
+ case CT_EXISTS:
+ siridb_group_decref(group);
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Group '%s' already exists.",
+ name);
+ break;
+
+ case CT_ERR:
+ siridb_group_decref(group);
+ sprintf(err_msg, "Memory allocation error.");
+ break;
+
+ case CT_OK:
+ if (vec_append_safe(&siridb->groups->ngroups, group))
+ {
+ siridb_group_decref(group);
+ sprintf(err_msg, "Memory allocation error.");
+ rc = -1;
+ }
+ else
+ {
+ siridb_group_incref(group);
+ }
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ uv_mutex_unlock(&siridb->groups->mutex);
+
+ return rc;
+}
+
+void siridb_groups_destroy(siridb_groups_t * groups)
+{
+ groups->status = GROUPS_STOPPING;
+}
+
+/*
+ * Main thread.
+ */
+int siridb_groups_save(siridb_groups_t * groups)
+{
+ qp_fpacker_t * fpacker;
+
+ log_debug("Write groups to file: '%s'", groups->fn);
+
+ return (
+ /* open a new user file */
+ (fpacker = qp_open(groups->fn, "w")) == NULL ||
+
+ /* open a new array */
+ qp_fadd_type(fpacker, QP_ARRAY_OPEN) ||
+
+ /* write the current schema */
+ qp_fadd_int64(fpacker, SIRIDB_GROUPS_SCHEMA) ||
+
+ /* we can and should skip this if we have no users to save */
+ ct_values(groups->groups, (ct_val_cb) GROUPS_write, fpacker) ||
+
+ /* close file pointer */
+ qp_close(fpacker)) ? EOF : 0;
+
+}
+
+/*
+ * Typedef: sirinet_clserver_get_file
+ *
+ * Returns the length of the content for a file and set buffer with the file
+ * content. Note that malloc is used to allocate memory for the buffer.
+ *
+ * In case of an error -1 is returned and buffer will be set to NULL.
+ */
+ssize_t siridb_groups_get_file(char ** buffer, siridb_t * siridb)
+{
+ return xpath_get_content(buffer, siridb->groups->fn);
+}
+
+/*
+ * Initialize each 'n' group property with the local value.
+ */
+void siridb_groups_init_nseries(siridb_groups_t * groups)
+{
+ ct_values(groups->groups, (ct_val_cb) GROUPS_nseries, NULL);
+}
+
+/*
+ * Main thread.
+ *
+ * Returns NULL and raises a signal in case of an error.
+ */
+sirinet_pkg_t * siridb_groups_pkg(siridb_groups_t * groups, uint16_t pid)
+{
+ qp_packer_t * packer = sirinet_packer_new(8192);
+ int rc;
+
+ if (packer == NULL || qp_add_type(packer, QP_ARRAY_OPEN))
+ {
+ return NULL; /* signal is raised */
+ }
+
+ rc = ct_values(groups->groups, (ct_val_cb) GROUPS_pkg, packer);
+
+ if (rc)
+ {
+ /* signal is raised when not 0 */
+ qp_packer_free(packer);
+ return NULL;
+ }
+
+ return sirinet_packer2pkg(packer, pid, BPROTO_RES_GROUPS);
+}
+
+/*
+ * Main thread.
+ *
+ * Returns 0 if successful or -1 when the group is not found.
+ * (in case not found an error message is set)
+ *
+ * Note: when saving the groups to disk has failed, we log critical but
+ * the function still returns 0;
+ */
+int siridb_groups_drop_group(
+ siridb_groups_t * groups,
+ const char * name,
+ char * err_msg)
+{
+ uv_mutex_lock(&groups->mutex);
+
+ siridb_group_t * group = (siridb_group_t *) ct_pop(groups->groups, name);
+
+ uv_mutex_unlock(&groups->mutex);
+
+ if (group == NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Group '%s' does not exist.",
+ name);
+ return -1;
+ }
+
+ group->flags |= GROUP_FLAG_DROPPED;
+ siridb_group_decref(group);
+
+ if (siridb_groups_save(groups))
+ {
+ log_critical("Cannot save groups to file: '%s'", groups->fn);
+ }
+
+ return 0;
+}
+
+/*
+ * Main thread.
+ */
+static int GROUPS_pkg(siridb_group_t * group, qp_packer_t * packer)
+{
+ int rc = 0;
+ rc += qp_add_type(packer, QP_ARRAY2);
+ rc += qp_add_string_term(packer, group->name);
+ rc += qp_add_int64(packer, group->series->len);
+ return rc;
+}
+
+/*
+ * Main thread.
+ */
+static int GROUPS_nseries(
+ siridb_group_t * group,
+ void * data __attribute__((unused)))
+{
+ group->n = group->series->len;
+ return 0;
+}
+
+void siridb__groups_free(siridb_groups_t * groups)
+{
+ size_t i;
+ free(groups->fn);
+
+ if (groups->nseries != NULL)
+ {
+ siridb_series_t * series;
+ for (i = 0; i < groups->nseries->len; i++)
+ {
+ series = (siridb_series_t *) groups->nseries->data[i];
+ siridb_series_decref(series);
+ }
+ vec_free(groups->nseries);
+ }
+
+ if (groups->groups != NULL)
+ {
+ ct_free(groups->groups, (ct_free_cb) siridb__group_decref);
+ }
+
+ if (groups->ngroups != NULL)
+ {
+ siridb_group_t * group;
+ for (i = 0; i < groups->ngroups->len; i++)
+ {
+ group = (siridb_group_t *) groups->ngroups->data[i];
+ siridb_group_decref(group);
+ }
+ vec_free(groups->ngroups);
+ }
+
+ uv_mutex_destroy(&groups->mutex);
+
+ free(groups);
+}
+
+/*
+ * Group thread.
+ */
+static int GROUPS_2vec(siridb_group_t * group, vec_t * groups_list)
+{
+ siridb_group_incref(group);
+ vec_append(groups_list, group);
+ return 0;
+}
+
+
+/*
+ * Group thread.
+ */
+static void GROUPS_loop(void * arg)
+{
+ siridb_t * siridb = arg;
+ siridb_groups_t * groups = siridb->groups;
+ uint64_t mod_test = 0;
+
+ siridb_groups_incref(siridb->groups);
+ siridb_tags_incref(siridb->tags);
+
+ while (groups->status != GROUPS_STOPPING)
+ {
+ sleep(GROUPS_LOOP_SLEEP);
+
+ if (groups->status == GROUPS_STOPPING)
+ break;
+
+ if ((siridb_is_reindexing(siridb) ||
+ siridb_server_self_synchronizing(siridb->server)) &&
+ (++mod_test % GROUPS_LOOP_DEEP))
+ {
+ /* less frequently when re-indexing or synchronizing */
+ continue;
+ }
+
+ switch((siridb_groups_status_t) groups->status)
+ {
+ case GROUPS_RUNNING:
+ /*
+ * Order of the steps below is important. Series must be handled
+ * before groups.
+ */
+ if (groups->nseries->len)
+ {
+ GROUPS_init_series(siridb);
+ }
+ if (groups->ngroups->len)
+ {
+ GROUPS_init_groups(siridb);
+ }
+ if (groups->flags & GROUPS_FLAG_DROPPED_SERIES)
+ {
+ GROUPS_cleanup(siridb->groups);
+ }
+ if (siridb->tags->flags & TAGS_FLAG_DROPPED_SERIES)
+ {
+ siridb_tags_dropped_series(siridb->tags);
+ }
+ if (siridb->tags->flags & TAGS_FLAG_REQUIRE_SAVE)
+ {
+ siridb_tags_save(siridb->tags);
+ }
+ break;
+
+ case GROUPS_STOPPING:
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+ }
+
+ groups->status = GROUPS_CLOSED;
+
+ siridb_tags_decref(siridb->tags);
+ siridb_groups_decref(siridb->groups);
+}
+
+static int GROUPS_load(siridb_t * siridb)
+{
+ int rc = 0;
+
+ if (!xpath_file_exist(siridb->groups->fn))
+ {
+ /* no groups file, create a new one */
+ return siridb_groups_save(siridb->groups);
+ }
+
+ qp_unpacker_t * unpacker = siridb_misc_open_schema_file(
+ SIRIDB_GROUPS_SCHEMA,
+ siridb->groups->fn);
+
+ if (unpacker == NULL)
+ {
+ rc = -1;
+ }
+ else
+ {
+ qp_obj_t qp_name, qp_source;
+ char err_msg[SIRIDB_MAX_SIZE_ERR_MSG];
+
+ while ( !rc &&
+ qp_is_array(qp_next(unpacker, NULL)) &&
+ qp_next(unpacker, &qp_name) == QP_RAW &&
+ qp_is_raw_term(&qp_name) &&
+ qp_next(unpacker, &qp_source) == QP_RAW)
+ {
+ rc = siridb_groups_add_group(
+ siridb,
+ (const char *) qp_name.via.raw,
+ (const char *) qp_source.via.raw,
+ qp_source.len,
+ err_msg);
+ }
+ qp_unpacker_ff_free(unpacker);
+ }
+
+ return rc;
+}
+
+/*
+ * Main thread.
+ */
+static int GROUPS_write(siridb_group_t * group, qp_fpacker_t * fpacker)
+{
+ int rc = 0;
+
+ rc += qp_fadd_type(fpacker, QP_ARRAY2);
+ rc += qp_fadd_raw(
+ fpacker,
+ (const unsigned char *) group->name,
+ strlen(group->name) + 1);
+ rc += qp_fadd_string(fpacker, group->source);
+
+ return rc;
+}
+
+/*
+ * Group thread.
+ */
+static void GROUPS_init_series(siridb_t * siridb)
+{
+ siridb_groups_t * groups = siridb->groups;
+ siridb_series_t * series;
+
+ uv_mutex_lock(&groups->mutex);
+
+ /* calculate modulo size [1..1001] */
+ int m = CALC_BATCH_SIZE(groups->groups->len);
+
+ while (groups->nseries->len)
+ {
+ series = (siridb_series_t *) vec_pop(groups->nseries);
+
+ if (~series->flags & SIRIDB_SERIES_IS_DROPPED)
+ {
+ ct_values(
+ groups->groups,
+ (ct_val_cb) siridb_group_test_series,
+ series);
+ }
+
+ siridb_series_decref(series);
+
+ if (groups->nseries->len % m == 0)
+ {
+ uv_mutex_unlock(&groups->mutex);
+
+ usleep(10000); /* 10ms */
+
+ uv_mutex_lock(&groups->mutex);
+
+ /* re-calculate modulo size [1..1001] */
+ m = CALC_BATCH_SIZE(groups->groups->len);
+ }
+ }
+
+ uv_mutex_unlock(&groups->mutex);
+}
+
+/*
+ * Group thread.
+ */
+static void GROUPS_init_groups(siridb_t * siridb)
+{
+ siridb_groups_t * groups = siridb->groups;
+ siridb_group_t * group;
+ vec_t * series_list;
+ siridb_series_t * series;
+ size_t i;
+
+ /* do not run this function when no groups need initialization */
+ assert (siridb->groups->ngroups->len);
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ series_list = (groups->nseries->len) ?
+ NULL : imap_2vec_ref(siridb->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (series_list == NULL)
+ {
+ log_info("Skip processing groups since new series are added.");
+ return;
+ }
+
+ uv_mutex_lock(&groups->mutex);
+
+ while (groups->ngroups->len)
+ {
+ group = (siridb_group_t *) vec_pop(groups->ngroups);
+
+ /* we must be sure this group is empty */
+ assert (group->series->len == 0);
+
+ if (~group->flags & GROUP_FLAG_DROPPED)
+ {
+ /* remove INIT flag from group */
+ group->flags &= ~GROUP_FLAG_INIT;
+
+ for (i = 0; i < series_list->len; i++)
+ {
+ series = (siridb_series_t *) series_list->data[i];
+
+ siridb_group_test_series(group, series);
+
+ if (i % GROUPS_RE_BATCH_SZ == 0)
+ {
+ uv_mutex_unlock(&groups->mutex);
+
+ usleep(10000); /* 10ms */
+
+ uv_mutex_lock(&groups->mutex);
+ }
+ }
+ }
+
+ siridb_group_decref(group);
+
+ uv_mutex_unlock(&groups->mutex);
+
+ usleep(10000); /* 10ms */
+
+ uv_mutex_lock(&groups->mutex);
+ }
+
+ uv_mutex_unlock(&groups->mutex);
+
+ for (i = 0; i < series_list->len; i++)
+ {
+ series = (siridb_series_t *) series_list->data[i];
+ siridb_series_decref(series);
+ }
+
+ vec_free(series_list);
+}
+
+/*
+ * Group thread.
+ */
+static void GROUPS_cleanup(siridb_groups_t * groups)
+{
+ uv_mutex_lock(&groups->mutex);
+
+ groups->flags &= ~GROUPS_FLAG_DROPPED_SERIES;
+
+ vec_t * groups_list = vec_new(groups->groups->len);
+
+ if (groups_list != NULL)
+ {
+ ct_values(groups->groups, (ct_val_cb) GROUPS_2vec, groups_list);
+ }
+
+ uv_mutex_unlock(&groups->mutex);
+
+ if (groups_list == NULL)
+ {
+ log_critical("Groups cleanup failed because of an allocation error.");
+ groups->flags |= GROUPS_FLAG_DROPPED_SERIES;
+ return;
+ }
+
+ siridb_group_t * group;
+
+ while (groups_list->len)
+ {
+ group = (siridb_group_t *) vec_pop(groups_list);
+
+ if (!group->flags)
+ {
+ uv_mutex_lock(&groups->mutex);
+
+ siridb_group_cleanup(group);
+
+ uv_mutex_unlock(&groups->mutex);
+ }
+
+ siridb_group_decref(group);
+
+ usleep(10000); /* 10ms */
+ }
+
+ vec_free(groups_list);
+}
--- /dev/null
+/*
+ * initsync.c - Initial replica synchronization.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <siri/db/initsync.h>
+#include <siri/err.h>
+#include <stddef.h>
+#include <siri/db/series.h>
+#include <siri/db/server.h>
+#include <assert.h>
+#include <unistd.h>
+#include <siri/net/protocol.h>
+#include <stdio.h>
+#include <siri/siri.h>
+#include <siri/optimize.h>
+#include <qpack/qpack.h>
+
+#define INITSYNC_SLEEP 100 /* 100 milliseconds * active tasks */
+#define INITSYNC_TIMEOUT 120000 /* 2 minutes */
+#define INITSYNC_RETRY 30000 /* 30 seconds */
+#define INITSYC_FN ".initsync"
+
+void siridb_initsync_fopen(siridb_initsync_t * initsync, const char * opentype);
+static int INITSYNC_create_cb(siridb_series_t * series, FILE * fp);
+static void INITSYNC_work(uv_timer_t * timer);
+static void INITSYNC_next_series_id(siridb_t * siridb);
+static int INITSYNC_unlink(siridb_initsync_t * initsync);
+static inline int INITSYNC_fn(siridb_t * siridb, siridb_initsync_t * initsync);
+static void INITSYNC_pause(siridb_replicate_t * replicate);
+static void INITSYNC_send(uv_timer_t * timer);
+static void INITSYNC_on_insert_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+
+static const long int SIZE2 = 2 * sizeof(uint32_t);
+static char sync_progress[30];
+
+
+/*
+ * Returns a pointer to initsync. If 'create_new' is zero and an initial
+ * synchronization file cannot be found, NULL is returned.
+ *
+ * In case of an error we also return NULL but in this case a SIGNAL is raised
+ */
+siridb_initsync_t * siridb_initsync_open(siridb_t * siridb, int create_new)
+{
+ siridb_initsync_t * initsync = malloc(sizeof(siridb_initsync_t));
+ if (initsync == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ initsync->fn = NULL;
+ initsync->fp = NULL;
+ initsync->next_series_id = NULL;
+ initsync->pkg_points = NULL;
+ initsync->pkg_tags = NULL;
+
+ if (INITSYNC_fn(siridb, initsync) < 0)
+ {
+ ERR_ALLOC
+ siridb_initsync_free(&initsync);
+ }
+ else
+ {
+ siridb_initsync_fopen(initsync, create_new ? "w+" : "r+");
+ if (initsync->fp == NULL)
+ {
+ siridb_initsync_free(&initsync);
+ }
+ else
+ {
+ initsync->next_series_id = malloc(sizeof(uint32_t));
+ if (initsync->next_series_id == NULL)
+ {
+ ERR_ALLOC
+ siridb_initsync_free(&initsync);
+ }
+ else if (create_new)
+ {
+ if (imap_walk(
+ siridb->series_map,
+ (imap_cb) INITSYNC_create_cb,
+ initsync->fp) || fflush(initsync->fp))
+ {
+ ERR_FILE
+ siridb_initsync_free(&initsync);
+ }
+ }
+ else
+ {
+ siridb_series_t * series;
+ uint32_t series_id;
+ while (fread(
+ &series_id,
+ sizeof(uint32_t),
+ 1,
+ initsync->fp) == 1)
+ {
+ series = imap_get(
+ siridb->series_map,
+ series_id);
+
+ if (series != NULL)
+ {
+ series->flags |= SIRIDB_SERIES_INIT_REPL;
+ }
+ }
+ }
+ if (initsync != NULL)
+ {
+ initsync->size = ftello(initsync->fp);
+ if (initsync->size == -1)
+ {
+ ERR_FILE
+ siridb_initsync_free(&initsync);
+ }
+ else if (initsync->size < (long int) sizeof(uint32_t))
+ {
+ log_warning("Empty synchronization file found...");
+ INITSYNC_unlink(initsync);
+ siridb_initsync_free(&initsync);
+ }
+ else
+ {
+ if (fseeko(initsync->fp, -sizeof(uint32_t), SEEK_END) ||
+ fread( initsync->next_series_id,
+ sizeof(uint32_t),
+ 1,
+ initsync->fp) != 1)
+ {
+ ERR_FILE
+ siridb_initsync_free(&initsync);
+ }
+ else
+ {
+ siri_optimize_pause();
+ }
+ }
+ }
+ }
+ }
+ }
+ return initsync;
+}
+
+/*
+ * Destroy *initsync and set initsync to NULL. Parsing *initsync == NULL is
+ * not allowed.
+ *
+ * (a SIGNAL is raised in case the file cannot be closed)
+ */
+void siridb_initsync_free(siridb_initsync_t ** initsync)
+{
+ if ((*initsync)->fp != NULL && fclose((*initsync)->fp))
+ {
+ ERR_FILE
+ }
+ free((*initsync)->fn);
+ free((*initsync)->next_series_id);
+ free((*initsync)->pkg_points);
+ free((*initsync)->pkg_tags);
+ free(*initsync);
+ *initsync = NULL;
+}
+
+/*
+ * Start initial replica work.
+ */
+void siridb_initsync_run(uv_timer_t * timer)
+{
+ siridb_t * siridb = (siridb_t *) timer->data;
+ uv_timer_start(
+ timer,
+ (siridb->replicate->initsync->pkg_points == NULL) ?
+ INITSYNC_work : INITSYNC_send,
+ INITSYNC_SLEEP * siridb->tasks.active,
+ 0);
+}
+
+/*
+ * Returns a human readable synchronization progress status
+ */
+const char * siridb_initsync_sync_progress(siridb_t * siridb)
+{
+ if (siridb->replica == NULL || siridb->replicate->initsync == NULL)
+ {
+ sprintf(sync_progress, "not available");
+ }
+ else
+ {
+ size_t num = siridb->replicate->initsync->size / sizeof(uint32_t);
+ size_t total = siridb->series_map->len;
+ double percent = 100 * (double) (total - num) / total;
+
+ sprintf(sync_progress,
+ "approximately at %0.2f%%",
+ (0 > percent) ? 0 : percent);
+ }
+ return sync_progress;
+}
+
+/*
+ * Open the initsync file. (set both the file pointer and file descriptor
+ * In case of and error, initsync->fp is set to NULL
+ */
+void siridb_initsync_fopen(siridb_initsync_t * initsync, const char * opentype)
+{
+ initsync->fp = fopen(initsync->fn, opentype);
+ if (initsync->fp != NULL)
+ {
+ initsync->fd = fileno(initsync->fp);
+ if (initsync->fd == -1)
+ {
+ log_critical("Error reading file descriptor: '%s'", initsync->fn);
+ fclose(initsync->fp);
+ initsync->fp = NULL;
+ }
+ }
+}
+
+/*
+ * Call-back function: sirinet_promise_cb
+ */
+static void INITSYNC_on_empty_tags_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ if (status)
+ {
+ log_error("Error while sending empty tags (%d)", status);
+ }
+ else if (sirinet_protocol_is_error(pkg->tp))
+ {
+ log_error(
+ "Error occurred while processing data on the new server: "
+ "(response type: %u)", pkg->tp);
+ }
+
+ sirinet_promise_decref(promise);
+}
+
+/*
+ * Read the next series id and truncate the synchronization file to remove
+ * the last synchronization id.
+ *
+ * This function might destroy 'replicate->initsync' when initial
+ * synchronization is finished.
+ */
+static void INITSYNC_next_series_id(siridb_t * siridb)
+{
+ assert (siridb->replicate != NULL);
+ assert (siridb->replicate->status == REPLICATE_RUNNING ||
+ siridb->replicate->status == REPLICATE_STOPPING);
+
+ siridb_initsync_t * initsync = siridb->replicate->initsync;
+
+ /* free the current package (can be NULL already) */
+ free(initsync->pkg_points);
+ free(initsync->pkg_tags);
+ initsync->pkg_points = NULL;
+ initsync->pkg_tags = NULL;
+
+ if (initsync->size >= SIZE2)
+ {
+ initsync->size -= sizeof(uint32_t);
+ if (fseeko(initsync->fp, -SIZE2, SEEK_END) ||
+ fread( initsync->next_series_id,
+ sizeof(uint32_t),
+ 1,
+ initsync->fp) != 1 ||
+ ftruncate(initsync->fd, initsync->size))
+ {
+ ERR_FILE
+ log_critical(
+ "Reading next series id has failed "
+ "(replicate status: %d)",
+ siridb->replicate->status);
+ }
+ else if (siridb->replicate->status == REPLICATE_STOPPING)
+ {
+ INITSYNC_pause(siridb->replicate);
+ }
+ else
+ {
+ siridb_initsync_run(siridb->replicate->timer);
+ }
+ }
+ else
+ {
+ sirinet_pkg_t * pkg;
+
+ /* send empty tags if required */
+ pkg = siridb_tags_empty(siridb->tags);
+ if (pkg)
+ {
+ if (siridb_server_send_pkg(
+ siridb->replica,
+ pkg,
+ INITSYNC_TIMEOUT,
+ (sirinet_promise_cb) INITSYNC_on_empty_tags_response,
+ NULL,
+ 0))
+ {
+ free(pkg);
+ }
+ }
+
+ log_info("Finished initial replica synchronization");
+ INITSYNC_unlink(initsync);
+ siridb_initsync_free(&siridb->replicate->initsync);
+
+ siridb->replicate->status =
+ (siridb->replicate->status == REPLICATE_STOPPING) ?
+ REPLICATE_PAUSED : REPLICATE_IDLE;
+ siridb_replicate_start(siridb->replicate);
+ siri_optimize_continue();
+ }
+}
+
+/*
+ * Close the initial synchronization file and set status to REPLICATE_PAUSED.
+ * (should only be called when status is REPLICATE_STOPPING)
+ */
+static void INITSYNC_pause(siridb_replicate_t * replicate)
+{
+ assert (replicate->status == REPLICATE_STOPPING);
+ if (fclose(replicate->initsync->fp))
+ {
+ log_critical("Error occurred while closing file: '%s'",
+ replicate->initsync->fn);
+ }
+ replicate->initsync->fp = NULL;
+ replicate->status = REPLICATE_PAUSED;
+}
+
+/*
+ * Type: uv_timer_cb
+ *
+ * This function sends a packed series to the replica server.
+ */
+static void INITSYNC_send(uv_timer_t * timer)
+{
+ siridb_t * siridb = (siridb_t *) timer->data;
+ assert (siridb->replicate->initsync->pkg_points != NULL);
+
+ if (siridb->replicate->status == REPLICATE_STOPPING)
+ {
+ INITSYNC_pause(siridb->replicate);
+ }
+ else
+ {
+ if (siridb_server_is_synchronizing(siridb->replica))
+ {
+ siridb_server_send_pkg(
+ siridb->replica,
+ siridb->replicate->initsync->pkg_points,
+ INITSYNC_TIMEOUT,
+ (sirinet_promise_cb) INITSYNC_on_insert_response,
+ siridb,
+ FLAG_KEEP_PKG);
+ }
+ else
+ {
+ log_info("Cannot send initial replica package to '%s' "
+ "(try again in %d seconds)",
+ siridb->replica->name,
+ INITSYNC_RETRY / 1000);
+ uv_timer_start(
+ siridb->replicate->timer,
+ INITSYNC_send,
+ INITSYNC_RETRY,
+ 0);
+ }
+ }
+}
+
+static void INITSYNC_work(uv_timer_t * timer)
+{
+ siridb_t * siridb = (siridb_t *) timer->data;
+
+ assert (siridb->replicate->status == REPLICATE_RUNNING ||
+ siridb->replicate->status == REPLICATE_STOPPING);
+ assert (siridb->replicate->initsync != NULL);
+ assert (siridb->replicate->initsync->fp != NULL);
+ assert (siridb->replicate->initsync->pkg_points == NULL);
+ assert (siridb->replicate->initsync->pkg_tags == NULL);
+
+ if (siridb->insert_tasks)
+ {
+ siridb_initsync_run(timer);
+ return;
+ }
+
+ siridb_initsync_t * initsync = siridb->replicate->initsync;
+ siridb_series_t * series;
+
+ series = imap_get(siridb->series_map, *initsync->next_series_id);
+
+ if (series != NULL)
+ {
+ uv_mutex_lock(&siridb->series_mutex);
+
+ siridb_points_t * points = siridb_series_get_points(
+ series,
+ NULL,
+ NULL);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (points != NULL)
+ {
+ qp_packer_t * packer = sirinet_packer_new(QP_SUGGESTED_SIZE);
+ if (packer != NULL)
+ {
+ qp_add_type(packer, QP_MAP1);
+
+ /* add name including string terminator */
+ qp_add_raw(
+ packer,
+ (const unsigned char *) series->name,
+ series->name_len + 1);
+
+ if (siridb_points_pack(points, packer) == 0)
+ {
+ series->flags &= ~SIRIDB_SERIES_INIT_REPL;
+
+ initsync->pkg_points = sirinet_packer2pkg(
+ packer,
+ 0,
+ BPROTO_INSERT_SERVER);
+
+ initsync->pkg_tags = siridb_tags_series(series);
+
+ uv_timer_start(
+ siridb->replicate->timer,
+ INITSYNC_send,
+ 0,
+ 0);
+
+ }
+ else
+ {
+ qp_packer_free(packer); /* signal is raised */
+ }
+ }
+ siridb_points_free(points);
+ }
+ }
+ else
+ {
+ INITSYNC_next_series_id(siridb);
+ }
+}
+
+/*
+ * Call-back function: sirinet_promise_cb
+ */
+static void INITSYNC_on_tag_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ if (status)
+ {
+ log_error("Error while sending tags (%d)", status);
+ }
+ else if (sirinet_protocol_is_error(pkg->tp))
+ {
+ log_error(
+ "Error occurred while processing data on the replica: "
+ "(response type: %u)", pkg->tp);
+ }
+
+ sirinet_promise_decref(promise);
+}
+
+/*
+ * Call-back function: sirinet_promise_cb
+ */
+static void INITSYNC_on_insert_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ siridb_t * siridb = (siridb_t *) promise->data;
+
+ switch ((sirinet_promise_status_t) status)
+ {
+ case PROMISE_WRITE_ERROR:
+ /*
+ * Write to socket error, data is not send so we should not commit.
+ */
+ uv_timer_start(
+ siridb->replicate->timer,
+ INITSYNC_send,
+ INITSYNC_RETRY,
+ 0);
+ break;
+ case PROMISE_TIMEOUT_ERROR:
+ /*
+ * Timeout occurred, use commit error since the data can be
+ * processed by the replica, we're just not sure.
+ */
+ case PROMISE_CANCELLED_ERROR:
+ /*
+ * Promise is cancelled but most likely the data is successful
+ * processed. Use siridb_fifo_commit_err() since we're not sure.
+ */
+ case PROMISE_PKG_TYPE_ERROR:
+ /*
+ * Commit with error since this package has result in an unknown
+ * package type.
+ */
+ log_error("Error occurred while sending series to the replica (%d)",
+ status);
+ /* TODO: maybe write pkg to an error queue ? */
+ INITSYNC_next_series_id(siridb);
+ break;
+ case PROMISE_SUCCESS:
+ if (sirinet_protocol_is_error(pkg->tp))
+ {
+ log_error(
+ "Error occurred while processing data on the replica: "
+ "(response type: %u)", pkg->tp);
+ /* TODO: maybe write pkg to an error queue ? */
+ }
+ if (siridb->replicate->initsync->pkg_tags)
+ {
+ siridb_server_send_pkg(
+ siridb->replica,
+ siridb->replicate->initsync->pkg_tags,
+ INITSYNC_TIMEOUT,
+ (sirinet_promise_cb) INITSYNC_on_tag_response,
+ NULL,
+ FLAG_KEEP_PKG);
+ }
+ INITSYNC_next_series_id(siridb);
+ break;
+ default:
+ assert (0);
+ break;
+ }
+ sirinet_promise_decref(promise);
+}
+
+
+/*
+ * Typedef: imap_cb
+ *
+ * Returns 0 if successful
+ */
+static int INITSYNC_create_cb(siridb_series_t * series, FILE * fp)
+{
+ series->flags |= SIRIDB_SERIES_INIT_REPL;
+ return fwrite(&series->id, sizeof(uint32_t), 1, fp) - 1;
+}
+
+/*
+ * Set initsync->fn to the correct file name.
+ *
+ * Returns the length of 'fn' or a negative value in case of an error.
+ */
+static inline int INITSYNC_fn(siridb_t * siridb, siridb_initsync_t * initsync)
+{
+ return asprintf(
+ &initsync->fn,
+ "%s%s",
+ siridb->dbpath,
+ INITSYC_FN);
+}
+
+/*
+ * Close the file pointer and remove the initial synchronization file.
+ * Return 0 if successful or -1 in case of an error.
+ */
+static int INITSYNC_unlink(siridb_initsync_t * initsync)
+{
+ assert (initsync->fp != NULL);
+ fclose(initsync->fp);
+ initsync->fp = NULL;
+
+ if (unlink(initsync->fn))
+ {
+ log_critical(
+ "Initial synchronization file could not be removed: '%s'",
+ initsync->fn);
+ return -1;
+ }
+ else
+ {
+ log_info("Initial synchronization file removed: '%s'", initsync->fn);
+ }
+ return 0;
+}
--- /dev/null
+/*
+ * insert.c - Handler database inserts.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <qpack/qpack.h>
+#include <siri/async.h>
+#include <siri/db/buffer.h>
+#include <siri/db/forward.h>
+#include <siri/db/insert.h>
+#include <siri/db/points.h>
+#include <siri/db/replicate.h>
+#include <siri/db/series.h>
+#include <siri/db/servers.h>
+#include <siri/err.h>
+#include <siri/net/promises.h>
+#include <siri/net/protocol.h>
+#include <siri/net/clserver.h>
+#include <siri/siri.h>
+#include <stdio.h>
+#include <string.h>
+#include <siri/db/tasks.h>
+
+#define MAX_INSERT_MSG 236
+#define INSERT_TIMEOUT 300000 /* 5 minutes */
+#define INSERT_AT_ONCE 3000 /* one point counts as 1, a series as 100 */
+#define WEIGHT_SERIES 50
+#define WEIGHT_NEW_SERIES 100
+
+#define SERIES_UPDATE_TS(series) \
+if (*ts < series->start) \
+{ \
+ series->start = *ts; \
+} \
+if (*ts > series->end) \
+{ \
+ series->end = *ts; \
+}
+
+static void INSERT_free(uv_handle_t * handle);
+static void INSERT_points_to_pools(uv_async_t * handle);
+static void INSERT_on_response(vec_t * promises, uv_async_t * handle);
+static uint16_t INSERT_get_pool(siridb_t * siridb, qp_obj_t * qp_series_name);
+
+static void INSERT_local_free_cb(uv_async_t * handle);
+static int8_t INSERT_local_work(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_obj_t * qp_series_name,
+ siridb_pcache_t ** pcache);
+static int INSERT_local_work_test(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_obj_t * qp_series_name,
+ siridb_pcache_t ** pcache,
+ siridb_forward_t ** forward);
+static void INSERT_local_task(uv_async_t * handle);
+static void INSERT_local_promise_cb(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+static void INSERT_local_promise_backend_cb(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+static int INSERT_init_local(
+ siridb_t * siridb,
+ sirinet_promises_t * promises,
+ sirinet_pkg_t * pkg,
+ uint8_t flags);
+
+static ssize_t INSERT_assign_by_map(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_packer_t * packer[]);
+
+static ssize_t INSERT_assign_by_array(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_packer_t * packer[],
+ qp_packer_t * tmp_packer);
+
+static int INSERT_read_points(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ qp_unpacker_t * unpacker,
+ qp_obj_t * qp_obj,
+ ssize_t * count);
+
+/*
+ * Return an error message for an insert err.
+ */
+const char * siridb_insert_err_msg(siridb_insert_err_t err)
+{
+ switch (err)
+ {
+ case ERR_EXPECTING_ARRAY:
+ return "Expecting an array with points.";
+ case ERR_EXPECTING_SERIES_NAME:
+ return "Expecting a series name (string value) with an array of "
+ "points where each point should be an integer time-stamp "
+ "with a value.";
+ case ERR_EXPECTING_MAP_OR_ARRAY:
+ return "Expecting an array or map containing series and points.";
+ case ERR_EXPECTING_INTEGER_TS:
+ return "Expecting an integer value as time-stamp.";
+ case ERR_TIMESTAMP_OUT_OF_RANGE:
+ return "Received at least one time-stamp which is out-of-range.";
+ case ERR_UNSUPPORTED_VALUE:
+ return "Unsupported value received. (only integer, float and string "
+ "values are supported).";
+ case ERR_EXPECTING_AT_LEAST_ONE_POINT:
+ return "Expecting a series to have at least one point.";
+ case ERR_EXPECTING_NAME_AND_POINTS:
+ return "Expecting a map with name and points.";
+ case ERR_INCOMPATIBLE_SERVER_VERSION:
+ return "At least one server is incompatible for handling this "
+ "insert.";
+ case ERR_MEM_ALLOC:
+ return "Critical memory allocation error";
+ default:
+ assert (0);
+ break;
+ }
+ return "Unknown err";
+}
+
+/*
+ * Destroy insert.
+ */
+void siridb_insert_free(siridb_insert_t * insert)
+{
+ size_t n;
+
+ /* free packer */
+ for (n = 0; n < insert->packer_size; n++)
+ {
+ if (insert->packer[n] != NULL)
+ {
+ qp_packer_free(insert->packer[n]);
+ }
+ }
+
+ /* free insert */
+ free(insert);
+}
+
+/*
+ * Returns a negative value in case of an error or a value equal to zero or
+ * higher representing the number of points processed.
+ *
+ * This function can set a SIGNAL when not enough space in the packer can be
+ * allocated for the points and ERR_MEM_ALLOC will be the return value if this
+ * is the case.
+ */
+ssize_t siridb_insert_assign_pools(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_packer_t * packer[])
+{
+ ssize_t rc = 0;
+ qp_types_t tp;
+ tp = qp_next(unpacker, NULL);
+
+ if (qp_is_map(tp))
+ {
+ rc = INSERT_assign_by_map(siridb, unpacker, packer);
+ }
+ else if (qp_is_array(tp))
+ {
+ qp_packer_t * tmp_packer = qp_packer_new(QP_SUGGESTED_SIZE);
+ if (tmp_packer == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ rc = INSERT_assign_by_array(
+ siridb,
+ unpacker,
+ packer,
+ tmp_packer);
+ qp_packer_free(tmp_packer);
+ }
+ }
+ else
+ {
+ rc = ERR_EXPECTING_MAP_OR_ARRAY;
+ }
+
+ return (siri_err) ? ERR_MEM_ALLOC : rc;
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+siridb_insert_t * siridb_insert_new(
+ siridb_t * siridb,
+ uint16_t pid,
+ sirinet_stream_t * client)
+{
+ siridb_insert_t * insert = malloc(
+ sizeof(siridb_insert_t) +
+ siridb->pools->len * sizeof(qp_packer_t *));
+
+ if (insert == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ size_t n;
+ uint32_t psize;
+
+ insert->free_cb = INSERT_free;
+ insert->ref = 1; /* used as reference on (siri_async_t) handle */
+
+ insert->flags = (siridb->flags & SIRIDB_FLAG_REINDEXING) ?
+ INSERT_FLAG_TEST : 0;
+
+ /* n-points will be set later to the correct value */
+ insert->npoints = 0;
+
+ /* save PID and client so we can respond to the client */
+ insert->pid = pid;
+ insert->client = client;
+
+ /*
+ * we keep the packer size because the number of pools might change and
+ * at this point the pool->len is equal to when the insert was received
+ */
+ insert->packer_size = siridb->pools->len;
+
+ /*
+ * Allocate packers for sending data to pools. we allocate smaller
+ * sizes in case we have a lot of pools.
+ */
+ psize = QP_SUGGESTED_SIZE / ((siridb->pools->len / 4) + 1);
+
+ for (n = 0; n < siridb->pools->len; n++)
+ {
+ if ((insert->packer[n] = sirinet_packer_new(psize)) == NULL)
+ {
+ return NULL; /* a signal is raised */
+ }
+
+ /* cannot raise a signal since enough space is allocated */
+ qp_add_type(insert->packer[n], QP_MAP_OPEN);
+ }
+ }
+ return insert;
+}
+
+/*
+ * Bind n-points to insert object, lock the client and start async task.
+ *
+ * Returns 0 if successful or -1 and a SIGNAL is raised in case of an error.
+ */
+int siridb_insert_points_to_pools(siridb_insert_t * insert, size_t npoints)
+{
+ uv_async_t * handle = malloc(sizeof(uv_async_t));
+ if (handle == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ /* bind the number of points to insert object */
+ insert->npoints= npoints;
+
+ /* increment the client reference counter */
+ sirinet_stream_incref(insert->client);
+
+ uv_async_init(siri.loop, handle, INSERT_points_to_pools);
+ handle->data = (void *) insert;
+
+ uv_async_send(handle);
+ return 0;
+}
+
+int insert_init_backend_local(
+ siridb_t * siridb,
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg,
+ uint8_t flags)
+{
+ sirinet_promise_t * promise = malloc(sizeof(sirinet_promise_t));
+ if (promise == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ siridb_insert_local_t * ilocal = malloc(sizeof(siridb_insert_local_t));
+ if (ilocal == NULL)
+ {
+ free(promise);
+ ERR_ALLOC
+ return -1;
+ }
+
+ uv_async_t * handle = malloc(sizeof(uv_async_t));
+ if (handle == NULL)
+ {
+ free(promise);
+ free(ilocal);
+ ERR_ALLOC
+ return -1;
+ }
+
+ ilocal->free_cb = (uv_close_cb) INSERT_local_free_cb;
+ ilocal->ref = 1;
+ ilocal->promise = promise;
+ ilocal->siridb = siridb;
+ ilocal->flags = flags;
+ ilocal->status = INSERT_LOCAL_CANCELLED;
+ ilocal->forward = NULL;
+ ilocal->pcache = NULL;
+
+ promise->pkg = sirinet_pkg_dup(pkg);
+ if (promise->pkg == NULL)
+ {
+ free(promise);
+ free(ilocal);
+ free(handle);
+ ERR_ALLOC
+ return -1;
+ }
+ qp_unpacker_init(&ilocal->unpacker, promise->pkg->data, promise->pkg->len);
+
+ sirinet_stream_incref(client);
+ promise->data = client;
+
+ promise->cb = (sirinet_promise_cb) INSERT_local_promise_backend_cb;
+ promise->pid = promise->pkg->pid;
+ promise->ref = 1;
+ promise->timer = NULL;
+ promise->server = NULL;
+
+ handle->data = ilocal;
+
+ qp_next(&ilocal->unpacker, NULL); /* map */
+ qp_next(&ilocal->unpacker, &ilocal->qp_series_name); /* first series/end */
+
+ siridb_tasks_inc(siridb->tasks);
+ siridb->insert_tasks++;
+
+ if (siridb_tee_is_connected(siridb->tee))
+ {
+ siridb_tee_write(siridb->tee, promise);
+ }
+
+ uv_async_init(siri.loop, handle, INSERT_local_task);
+ uv_async_send(handle);
+
+ return 0;
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * This function can raise a SIGNAL.
+ */
+static void INSERT_on_response(vec_t * promises, uv_async_t * handle)
+{
+ if (promises != NULL)
+ {
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ siridb_insert_t * insert = (siridb_insert_t *) handle->data;
+ siridb_t * siridb = insert->client->siridb;
+
+ int n = 0;
+ char msg[MAX_INSERT_MSG];
+
+ /* the packer size is big enough to hold MAX_INSERT_MSG + some overhead
+ * for creating the QPack message */
+ qp_packer_t * packer = sirinet_packer_new(256);
+
+ if (packer != NULL)
+ {
+ cproto_server_t tp = CPROTO_RES_INSERT;
+ size_t i;
+ sirinet_pkg_t * response_pkg;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+ if (siri_err || promise == NULL)
+ {
+ n = snprintf(msg,
+ MAX_INSERT_MSG,
+ "Critical error occurred on '%s'",
+ siridb->server->name);
+ tp = CPROTO_ERR_INSERT;
+ continue;
+ }
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg == NULL || pkg->tp != BPROTO_ACK_INSERT)
+ {
+ n = snprintf(msg,
+ MAX_INSERT_MSG,
+ "Error occurred while sending points to at "
+ "least '%s'",
+ promise->server->name);
+ tp = CPROTO_ERR_INSERT;
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+
+ /* this will fit for sure */
+ qp_add_type(packer, QP_MAP_OPEN);
+
+ if (tp == CPROTO_ERR_INSERT)
+ {
+ qp_add_raw(packer, (const unsigned char *) "error_msg", 9);
+ }
+ else
+ {
+ qp_add_raw(packer, (const unsigned char *) "success_msg", 11);
+ n = snprintf(msg,
+ MAX_INSERT_MSG,
+ "Successfully inserted %zd point(s).",
+ insert->npoints);
+ log_info(msg);
+ siridb->received_points += insert->npoints;
+ }
+
+ qp_add_raw(
+ packer,
+ (const unsigned char *) msg,
+ (n < MAX_INSERT_MSG) ? n : MAX_INSERT_MSG);
+
+ response_pkg = sirinet_packer2pkg(
+ packer,
+ insert->pid,
+ tp);
+
+ sirinet_pkg_send(insert->client, response_pkg);
+ }
+ }
+
+ uv_close((uv_handle_t *) handle, siri_async_close);
+}
+
+static void INSERT_local_free_cb(uv_async_t * handle)
+{
+ siridb_insert_local_t * ilocal = (siridb_insert_local_t *) handle->data;
+
+ /* this destroys the pkg and unpacker */
+ free(ilocal->promise->pkg);
+
+ if (ilocal->forward != NULL)
+ {
+ uv_async_t * fwd = malloc(sizeof(uv_async_t));
+ if (fwd == NULL || siri_err)
+ {
+ if (fwd == NULL)
+ {
+ ERR_ALLOC
+ }
+ ilocal->status = INSERT_LOCAL_ERROR;
+ siridb_forward_free(ilocal->forward);
+ }
+ else
+ {
+ uv_async_init(siri.loop, fwd, siridb_forward_points_to_pools);
+ fwd->data = (void *) ilocal->forward;
+ uv_async_send(fwd);
+ }
+ }
+
+ ilocal->promise->cb(ilocal->promise, NULL, ilocal->status);
+
+ siridb_tasks_dec(ilocal->siridb->tasks);
+ ilocal->siridb->insert_tasks--;
+ if (ilocal->pcache != NULL)
+ {
+ siridb_pcache_free(ilocal->pcache);
+ }
+ free(ilocal);
+ free(handle);
+}
+
+/*
+ * Returns insert->status
+ */
+static int8_t INSERT_local_work(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_obj_t * qp_series_name,
+ siridb_pcache_t ** pcache)
+{
+ qp_types_t tp;
+ siridb_series_t * series;
+ qp_obj_t qp_series_ts;
+ qp_obj_t qp_series_val;
+ qp_via_t forstr;
+ qp_via_t * val;
+ uint64_t * ts;
+ int n = INSERT_AT_ONCE;
+
+ /*
+ * we check for siri_err because siridb_series_add_point()
+ * should never be called twice on the same series after an
+ * error has occurred.
+ */
+ while ( !siri_err &&
+ qp_is_raw_term(qp_series_name) &&
+ qp_series_name->via.raw[0] != '\0' &&
+ (n -= WEIGHT_SERIES) > 0)
+ {
+ series = (siridb_series_t *) ct_get(
+ siridb->series,
+ (const char *) qp_series_name->via.raw);
+
+ qp_next(unpacker, NULL); /* array open */
+ qp_next(unpacker, NULL); /* first point array2 */
+ qp_next(unpacker, &qp_series_ts); /* first ts */
+ qp_next(unpacker, &qp_series_val); /* first val */
+
+ if (series == NULL)
+ {
+ series = siridb_series_new(
+ siridb,
+ (const char *) qp_series_name->via.raw,
+ SIRIDB_QP_MAP2_TP(qp_series_val.tp));
+
+ if (series == NULL)
+ {
+ log_critical(
+ "Error creating series: '%s'",
+ qp_series_name->via.raw);
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+
+ n -= WEIGHT_NEW_SERIES;
+ }
+
+ ts = (uint64_t *) &qp_series_ts.via.int64;
+ SERIES_UPDATE_TS(series)
+
+ siridb_series_ensure_type(series, &qp_series_val);
+ if ((tp = qp_next(unpacker, qp_series_name)) != QP_ARRAY2 &&
+ series->buffer != NULL)
+ {
+ if (siridb_series_add_point(
+ siridb,
+ series,
+ ts,
+ &qp_series_val.via))
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+ }
+ else
+ {
+ if (*pcache == NULL)
+ {
+ *pcache = siridb_pcache_new(series->tp);
+ if (*pcache == NULL)
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+ }
+ else
+ {
+ (*pcache)->tp = series->tp;
+ (*pcache)->len = 0;
+ }
+
+ if (series->tp == TP_STRING)
+ {
+ val = &forstr;
+ val->str = strndup(qp_series_val.via.str, qp_series_val.len);
+ if (val->str == NULL)
+ {
+ ERR_ALLOC
+ return INSERT_LOCAL_ERROR;
+ }
+ }
+ else
+ {
+ val = &qp_series_val.via;
+ }
+
+ /* this point will always fit */
+ siridb_pcache_add_point(*pcache, ts, val);
+
+ if (tp == QP_ARRAY2) do
+ {
+ qp_next(unpacker, &qp_series_ts); /* ts */
+ qp_next(unpacker, &qp_series_val); /* val */
+ siridb_series_ensure_type(series, &qp_series_val);
+
+ if (series->tp == TP_STRING)
+ {
+ val->str = \
+ strndup(qp_series_val.via.str, qp_series_val.len);
+ if (val->str == NULL)
+ {
+ ERR_ALLOC
+ return INSERT_LOCAL_ERROR;
+ }
+ }
+
+ ts = (uint64_t *) &qp_series_ts.via.int64;
+ SERIES_UPDATE_TS(series)
+
+ if (siridb_pcache_add_point(
+ *pcache,
+ ts,
+ val))
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+
+ n--;
+ }
+ while ((tp = qp_next(unpacker, qp_series_name)) == QP_ARRAY2);
+
+ if (siridb_series_add_pcache(
+ siridb,
+ series,
+ *pcache))
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+
+ if ((*pcache)->tp == TP_STRING)
+ {
+ siridb_points_free((siridb_points_t *) *pcache);
+ *pcache = NULL;
+ }
+ }
+
+ if (series->length == 0)
+ {
+ if (siridb_series_drop(siridb, series))
+ {
+ siridb_series_flush_dropped(siridb);
+ }
+ }
+
+ if (tp == QP_ARRAY_CLOSE)
+ {
+ qp_next(unpacker, qp_series_name);
+ }
+ }
+
+ return siri_err; /* expected to be 0 */
+}
+
+/*
+ * Returns insert->status
+ */
+static int INSERT_local_work_test(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_obj_t * qp_series_name,
+ siridb_pcache_t ** pcache,
+ siridb_forward_t ** forward)
+{
+ qp_types_t tp;
+ siridb_series_t * series;
+ uint16_t pool;
+ uint64_t * ts;
+ const char * series_name;
+ unsigned char * pt;
+ qp_obj_t qp_series_ts;
+ qp_obj_t qp_series_val;
+ qp_via_t forstr;
+ qp_via_t * val;
+ int n = INSERT_AT_ONCE;
+
+ /*
+ * we check for siri_err because siridb_series_add_point()
+ * should never be called twice on the same series after an
+ * error has occurred.
+ */
+ while ( !siri_err &&
+ qp_is_raw_term(qp_series_name) &&
+ (n -= WEIGHT_SERIES) > 0)
+ {
+ series_name = (char *) qp_series_name->via.raw;
+ series = (siridb_series_t *) ct_get(siridb->series, series_name);
+ if (series == NULL)
+ {
+ /* the series does not exist so check what to do... */
+ pool = siridb_lookup_sn(siridb->pools->lookup, series_name);
+
+ if (pool == siridb->server->pool)
+ {
+ /*
+ * This is the correct pool so create the series and
+ * add the points.
+ */
+
+ /* save pointer position and read series type */
+ pt = unpacker->pt;
+ qp_next(unpacker, NULL); /* array open */
+ qp_next(unpacker, NULL); /* first point array2 */
+ qp_next(unpacker, NULL); /* first ts */
+ qp_next(unpacker, &qp_series_val); /* first val */
+
+ /* restore pointer position */
+ unpacker->pt = pt;
+
+ series = siridb_series_new(
+ siridb,
+ series_name,
+ SIRIDB_QP_MAP2_TP(qp_series_val.tp));
+
+ if (series == NULL)
+ {
+ ERR_ALLOC
+ log_critical("Error creating series: '%s'", series_name);
+ return INSERT_LOCAL_ERROR;
+ }
+
+ n -= WEIGHT_NEW_SERIES;
+ }
+ else if (siridb->replica == NULL ||
+ siridb_series_server_id_by_name(series_name) ==
+ siridb->server->id)
+ {
+ /*
+ * Forward the series to the correct pool because 'this' server
+ * is responsible for the series.
+ */
+ if (*forward == NULL)
+ {
+ if ((*forward = siridb_forward_new(siridb)) == NULL)
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+ }
+ /* testing is not needed since we check for siri_err later */
+ qp_add_raw(
+ (*forward)->packer[pool],
+ (const unsigned char *) series_name,
+ qp_series_name->len);
+ qp_packer_extend_fu((*forward)->packer[pool], unpacker);
+ qp_next(unpacker, qp_series_name);
+ continue; /* expected to be 0 */
+ }
+ else
+ {
+ /*
+ * Skip this series since it will forwarded to the correct
+ * pool by the replica server.
+ */
+ qp_skip_next(unpacker); /* array */
+ qp_next(unpacker, qp_series_name);
+ continue; /* expected to be 0 */
+ }
+ }
+
+ qp_next(unpacker, NULL); /* array open */
+ qp_next(unpacker, NULL); /* first point array2 */
+ qp_next(unpacker, &qp_series_ts); /* first ts */
+ qp_next(unpacker, &qp_series_val); /* first val */
+
+ ts = (uint64_t *) &qp_series_ts.via.int64;
+ SERIES_UPDATE_TS(series)
+
+ siridb_series_ensure_type(series, &qp_series_val);
+
+ if ((tp = qp_next(unpacker, qp_series_name)) != QP_ARRAY2 &&
+ series->buffer != NULL)
+ {
+ if (siridb_series_add_point(
+ siridb,
+ series,
+ ts,
+ &qp_series_val.via))
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+ }
+ else
+ {
+ if (*pcache == NULL)
+ {
+ *pcache = siridb_pcache_new(series->tp);
+ if (*pcache == NULL)
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+ }
+ else
+ {
+ (*pcache)->tp = series->tp;
+ (*pcache)->len = 0;
+ }
+
+ if (series->tp == TP_STRING)
+ {
+ val = &forstr;
+ val->str = strndup(qp_series_val.via.str, qp_series_val.len);
+ if (val->str == NULL)
+ {
+ ERR_ALLOC
+ return INSERT_LOCAL_ERROR;
+ }
+ }
+ else
+ {
+ val = &qp_series_val.via;
+ }
+
+ siridb_pcache_add_point(*pcache, ts, val);
+
+ if (tp == QP_ARRAY2) do
+ {
+ qp_next(unpacker, &qp_series_ts); /* ts */
+ qp_next(unpacker, &qp_series_val); /* val */
+ siridb_series_ensure_type(series, &qp_series_val);
+
+ if (series->tp == TP_STRING)
+ {
+ val->str = \
+ strndup(qp_series_val.via.str, qp_series_val.len);
+ if (val->str == NULL)
+ {
+ ERR_ALLOC
+ return INSERT_LOCAL_ERROR;
+ }
+ }
+
+ ts = (uint64_t *) &qp_series_ts.via.int64;
+ SERIES_UPDATE_TS(series)
+
+ if (siridb_pcache_add_point(
+ *pcache,
+ ts,
+ val))
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+
+ n--;
+ }
+ while ((tp = qp_next(unpacker, qp_series_name)) == QP_ARRAY2);
+
+ if (siridb_series_add_pcache(
+ siridb,
+ series,
+ *pcache))
+ {
+ return INSERT_LOCAL_ERROR; /* signal is raised */
+ }
+
+ if ((*pcache)->tp == TP_STRING)
+ {
+ siridb_points_free((siridb_points_t *) *pcache);
+ *pcache = NULL;
+ }
+
+ }
+
+ if (series->length == 0)
+ {
+ if (siridb_series_drop(siridb, series))
+ {
+ siridb_series_flush_dropped(siridb);
+ }
+ }
+
+ if (tp == QP_ARRAY_CLOSE)
+ {
+ qp_next(unpacker, qp_series_name);
+ }
+ }
+
+ return siri_err; /* expected to be 0 */
+}
+
+static void INSERT_local_task(uv_async_t * handle)
+{
+
+ siridb_insert_local_t * ilocal = (siridb_insert_local_t *) handle->data;
+ qp_unpacker_t * unpacker = &ilocal->unpacker;
+ siridb_t * siridb;
+
+ /*
+ * we check for siri_err because siridb_series_add_point()
+ * should never be called twice on the same series after an
+ * error has occurred.
+ */
+ if (ilocal->status == INSERT_LOCAL_ERROR)
+ {
+ uv_close((uv_handle_t *) handle, siri_async_close);
+ return;
+ }
+
+ if (!qp_is_raw_term(&ilocal->qp_series_name))
+ {
+ ilocal->status = INSERT_LOCAL_SUCESS;
+ uv_close((uv_handle_t *) handle, siri_async_close);
+ return;
+ }
+
+ siridb = ilocal->siridb;
+
+ if (siridb->buffer->fp == NULL && siridb_buffer_open(siridb->buffer))
+ {
+ ERR_FILE
+ ilocal->status = INSERT_LOCAL_ERROR;
+ uv_close((uv_handle_t *) handle, siri_async_close);
+ return;
+ }
+
+ if (siridb->store == NULL && siridb_series_open_store(siridb))
+ {
+ ERR_FILE
+ ilocal->status = INSERT_LOCAL_ERROR;
+ uv_close((uv_handle_t *) handle, siri_async_close);
+ return;
+ }
+
+ uv_mutex_lock(&siridb->series_mutex);
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ if ((ilocal->flags & INSERT_FLAG_TEST) || (
+ (siridb->flags & SIRIDB_FLAG_REINDEXING) &&
+ (~ilocal->flags & INSERT_FLAG_TESTED)))
+ {
+ /*
+ * We can use INSERT_local_work_test even if 'this' server has not set
+ * the REINDEXING flag yet, since this does not depend on 'prev_lookup'
+ */
+ if (INSERT_local_work_test(
+ siridb,
+ unpacker,
+ &ilocal->qp_series_name,
+ &ilocal->pcache,
+ &ilocal->forward))
+ {
+ ilocal->status = INSERT_LOCAL_ERROR;
+ }
+ }
+ else
+ {
+ /* siri_err is raised in case of an error */
+ if (INSERT_local_work(
+ siridb,
+ unpacker,
+ &ilocal->qp_series_name,
+ &ilocal->pcache))
+ {
+ ilocal->status = INSERT_LOCAL_ERROR;
+ }
+ }
+
+ if (siri.buffersync == NULL)
+ {
+ if (siridb_buffer_fsync(siridb->buffer))
+ {
+ log_critical("fsync() has failed on the buffer file");
+ }
+ }
+
+ uv_mutex_unlock(&siridb->series_mutex);
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ uv_async_send(handle);
+}
+
+static void INSERT_local_promise_cb(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ assert (pkg == NULL);
+ pkg = sirinet_pkg_new(
+ 0,
+ 0,
+ status == 0 ? BPROTO_ACK_INSERT : BPROTO_ERR_INSERT,
+ NULL);
+ sirinet_promises_t * promises = (sirinet_promises_t *) promise->data;
+ promise->data = pkg;
+ vec_append(promises->promises, (void *) promise);
+
+ SIRINET_PROMISES_CHECK(promises)
+}
+
+static void INSERT_local_promise_backend_cb(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ assert (pkg == NULL);
+ sirinet_stream_t * client = promise->data;
+
+ pkg = sirinet_pkg_new(
+ promise->pid,
+ 0,
+ status == 0 ? BPROTO_ACK_INSERT : BPROTO_ERR_INSERT,
+ NULL);
+
+ if (pkg != NULL)
+ {
+ sirinet_pkg_send(client, pkg);
+ }
+ sirinet_stream_decref(client);
+ sirinet_promise_decref(promise);
+}
+
+/*
+ * Start async insert task.
+ *
+ * This function is responsible for calling free on pkg.
+ */
+static int INSERT_init_local(
+ siridb_t * siridb,
+ sirinet_promises_t * promises,
+ sirinet_pkg_t * pkg,
+ uint8_t flags)
+{
+ sirinet_promise_t * promise = malloc(sizeof(sirinet_promise_t));
+ if (promise == NULL)
+ {
+ free(pkg);
+ ERR_ALLOC
+ return -1;
+ }
+ siridb_insert_local_t * ilocal = malloc(sizeof(siridb_insert_local_t));
+ if (ilocal == NULL)
+ {
+ free(pkg);
+ free(promise);
+ ERR_ALLOC
+ return -1;
+ }
+
+ uv_async_t * handle = malloc(sizeof(uv_async_t));
+ if (handle == NULL)
+ {
+ free(pkg);
+ free(promise);
+ free(ilocal);
+ ERR_ALLOC
+ return -1;
+ }
+
+ ilocal->free_cb = (uv_close_cb) INSERT_local_free_cb;
+ ilocal->ref = 1;
+ ilocal->promise = promise;
+ ilocal->siridb = siridb;
+ qp_unpacker_init(&ilocal->unpacker, pkg->data, pkg->len);
+ ilocal->flags = flags;
+ ilocal->status = INSERT_LOCAL_CANCELLED;
+ ilocal->forward = NULL;
+ ilocal->pcache = NULL;
+
+ promise->pkg = pkg;
+ promise->data = promises;
+ /* We do not need an increment here since this is the local server */
+ promise->server = siridb->server;
+ promise->cb = (sirinet_promise_cb) INSERT_local_promise_cb;
+ promise->pid = 0;
+ promise->ref = 1;
+ promise->timer = NULL;
+
+ handle->data = ilocal;
+
+ qp_next(&ilocal->unpacker, NULL); /* map */
+ qp_next(&ilocal->unpacker, &ilocal->qp_series_name); /* first series/end */
+
+ siridb_tasks_inc(siridb->tasks);
+ siridb->insert_tasks++;
+
+ if (siridb_tee_is_connected(siridb->tee))
+ {
+ siridb_tee_write(siridb->tee, promise);
+ }
+
+ uv_async_init(siri.loop, handle, INSERT_local_task);
+ uv_async_send(handle);
+
+ return 0;
+}
+
+
+
+/*
+ * Call-back function: uv_async_cb
+ *
+ * In case of an error a SIGNAL is raised and a successful message will not
+ * be send to the client.
+ */
+static void INSERT_points_to_pools(uv_async_t * handle)
+{
+ siridb_insert_t * insert = (siridb_insert_t *) handle->data;
+ siridb_t * siridb = insert->client->siridb;
+
+ uint16_t pool = siridb->server->pool;
+ sirinet_pkg_t * pkg, * repl_pkg;
+ sirinet_promises_t * promises = sirinet_promises_new(
+ siridb->pools->len,
+ (sirinet_promises_cb) INSERT_on_response,
+ handle,
+ NULL);
+ int pool_count = 0;
+ uint16_t n;
+
+ if (promises == NULL)
+ {
+ return; /* signal is raised */
+ }
+
+ for (n = 0; n < insert->packer_size; n++)
+ {
+ if (insert->packer[n]->len == sizeof(sirinet_pkg_t) + 1)
+ {
+ /*
+ * skip empty packer.
+ * (empty packer has only sizeof(sirinet_pkg_t) + QP_MAP_OPEN)
+ */
+ qp_packer_free(insert->packer[n]);
+ }
+ else if (n == pool)
+ {
+ if (siridb->replica != NULL)
+ {
+ repl_pkg = siridb->replicate->initsync == NULL ? NULL :
+ siridb_replicate_pkg_filter(
+ siridb,
+ insert->packer[n]->buffer + sizeof(sirinet_pkg_t),
+ insert->packer[n]->len - sizeof(sirinet_pkg_t),
+ insert->flags);
+
+ pkg = sirinet_packer2pkg(
+ insert->packer[n],
+ 0,
+ (insert->flags & INSERT_FLAG_TEST) ?
+ BPROTO_INSERT_TEST_SERVER :
+ (insert->flags & INSERT_FLAG_TESTED) ?
+ BPROTO_INSERT_TESTED_SERVER :
+ BPROTO_INSERT_SERVER);
+
+ /* send to replica, use repl_pkg if needed */
+ siridb_replicate_pkg(
+ siridb,
+ repl_pkg == NULL ? pkg : repl_pkg);
+
+ free(repl_pkg);
+ }
+ else
+ {
+ pkg = sirinet_packer2pkg(insert->packer[n], 0, 0);
+ }
+
+ if (INSERT_init_local(
+ siridb,
+ promises,
+ pkg,
+ insert->flags) == 0)
+ {
+ pool_count++;
+ }
+ }
+ else
+ {
+ pkg = sirinet_packer2pkg(
+ insert->packer[n],
+ 0,
+ (insert->flags & INSERT_FLAG_TEST) ?
+ BPROTO_INSERT_TEST_POOL : BPROTO_INSERT_POOL);
+ if (siridb_pool_send_pkg(
+ siridb->pools->pool + n,
+ pkg,
+ INSERT_TIMEOUT,
+ (sirinet_promise_cb) sirinet_promises_on_response,
+ promises,
+ 0))
+ {
+ free(pkg);
+ log_error(
+ "Although we have checked and validated each pool "
+ "had at least one server available, it seems that the "
+ "situation has changed and we cannot send points to "
+ "pool %u", n);
+ }
+ else
+ {
+ pool_count++;
+ }
+ }
+ insert->packer[n] = NULL;
+ }
+
+ /* pool_count is always smaller than the initial promises->size */
+ promises->promises->size = pool_count;
+
+ SIRINET_PROMISES_CHECK(promises)
+}
+
+/*
+ * Returns the correct pool.
+ */
+static uint16_t INSERT_get_pool(siridb_t * siridb, qp_obj_t * qp_series_name)
+{
+ uint16_t pool;
+
+ if (~siridb->flags & SIRIDB_FLAG_REINDEXING)
+ {
+ /* when not re-indexing, select the correct pool */
+ pool = siridb_lookup_sn_raw(
+ siridb->pools->lookup,
+ (const char *) qp_series_name->via.raw,
+ qp_series_name->len);
+ }
+ else
+ {
+ if (ct_getn(
+ siridb->series,
+ (const char *) qp_series_name->via.raw,
+ qp_series_name->len) != NULL)
+ {
+ /*
+ * we are re-indexing and at least at this moment still own the
+ * series
+ */
+ pool = siridb->server->pool;
+ }
+ else
+ {
+ /*
+ * We are re-indexing and do not have the series.
+ * Select the correct pool BEFORE re-indexing was
+ * started or the new correct pool if this pool is
+ * the previous correct pool. (we can do this now
+ * because we known we don't have the series)
+ */
+ assert (siridb->pools->prev_lookup != NULL);
+ pool = siridb_lookup_sn_raw(
+ siridb->pools->prev_lookup,
+ (const char *) qp_series_name->via.raw,
+ qp_series_name->len);
+
+ if (pool == siridb->server->pool)
+ {
+ pool = siridb_lookup_sn_raw(
+ siridb->pools->lookup,
+ (const char *) qp_series_name->via.raw,
+ qp_series_name->len);
+ }
+ }
+ }
+ return pool;
+}
+
+/*
+ * Returns a negative value in case of an error or a value equal to zero or
+ * higher representing the number of points processed.
+ *
+ * This function can set a SIGNAL when not enough space in the packer can be
+ * allocated for the points and should be checked with 'siri_err'.
+ */
+static ssize_t INSERT_assign_by_map(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_packer_t * packer[])
+{
+ int tp; /* use int instead of qp_types_t for negative values */
+ uint16_t pool;
+ ssize_t count = 0;
+ qp_obj_t qp_obj;
+
+ tp = qp_next(unpacker, &qp_obj);
+
+ while ( tp == QP_RAW &&
+ qp_obj.len &&
+ qp_obj.len < SIRIDB_SERIES_NAME_LEN_MAX)
+ {
+ pool = INSERT_get_pool(siridb, &qp_obj);
+
+ qp_add_raw_term(packer[pool],
+ qp_obj.via.raw,
+ qp_obj.len);
+
+ if ((tp = INSERT_read_points(
+ siridb,
+ packer[pool],
+ unpacker,
+ &qp_obj,
+ &count)) < 0)
+ {
+ return tp;
+ }
+ }
+
+ if (tp != QP_END && tp != QP_MAP_CLOSE)
+ {
+ return ERR_EXPECTING_SERIES_NAME;
+ }
+
+ return count;
+}
+
+/*
+ * Returns a negative value in case of an error or a value equal to zero or
+ * higher representing the number of points processed.
+ *
+ * This function can set a SIGNAL when not enough space in the packer can be
+ * allocated for the points and should be checked with 'siri_err'.
+ */
+static ssize_t INSERT_assign_by_array(
+ siridb_t * siridb,
+ qp_unpacker_t * unpacker,
+ qp_packer_t * packer[],
+ qp_packer_t * tmp_packer)
+{
+ int tp; /* use int instead of qp_types_t for negative values */
+ uint16_t pool;
+ ssize_t count = 0;
+ qp_obj_t qp_obj;
+ tp = qp_next(unpacker, &qp_obj);
+
+ while (tp == QP_MAP2)
+ {
+ if (qp_next(unpacker, &qp_obj) != QP_RAW)
+ {
+ return ERR_EXPECTING_NAME_AND_POINTS;
+ }
+
+ if (strncmp((const char *) qp_obj.via.raw, "points", qp_obj.len) == 0)
+ {
+ if ((tp = INSERT_read_points(
+ siridb,
+ tmp_packer,
+ unpacker,
+ &qp_obj,
+ &count)) < 0 || tp != QP_RAW)
+ {
+ return (tp < 0) ? tp : ERR_EXPECTING_NAME_AND_POINTS;
+ }
+ }
+
+ if (strncmp((const char *) qp_obj.via.raw, "name", qp_obj.len) == 0)
+ {
+ if ( qp_next(unpacker, &qp_obj) != QP_RAW ||
+ !qp_obj.len ||
+ qp_obj.len >= SIRIDB_SERIES_NAME_LEN_MAX)
+ {
+ return ERR_EXPECTING_NAME_AND_POINTS;
+ }
+
+ pool = INSERT_get_pool(siridb, &qp_obj);
+
+ qp_add_raw_term(packer[pool],
+ qp_obj.via.raw,
+ qp_obj.len);
+ }
+ else
+ {
+ return ERR_EXPECTING_NAME_AND_POINTS;
+ }
+
+ if (tmp_packer->len)
+ {
+ qp_packer_extend(packer[pool], tmp_packer);
+ tmp_packer->len = 0;
+ tp = qp_next(unpacker, &qp_obj);
+ }
+ else
+ {
+ if (qp_next(unpacker, &qp_obj) != QP_RAW ||
+ strncmp(
+ (const char *) qp_obj.via.raw, "points", qp_obj.len))
+ {
+ return ERR_EXPECTING_NAME_AND_POINTS;
+ }
+
+ if ((tp = INSERT_read_points(
+ siridb,
+ packer[pool],
+ unpacker,
+ &qp_obj,
+ &count)) < 0)
+ {
+ return tp;
+ }
+ }
+ }
+
+ if (tp != QP_END && tp != QP_ARRAY_CLOSE)
+ {
+ return ERR_EXPECTING_SERIES_NAME;
+ }
+
+ return count;
+}
+
+/*
+ * Returns a negative value in case of an error or a value equal to zero or
+ * higher representing the next qpack type in the unpaker.
+ *
+ * This function can set a SIGNAL when not enough space in the packer can be
+ * allocated for the points.
+ */
+static int INSERT_read_points(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ qp_unpacker_t * unpacker,
+ qp_obj_t * qp_obj,
+ ssize_t * count)
+{
+ qp_types_t tp;
+
+ if (!qp_is_array(qp_next(unpacker, NULL)))
+ {
+ return ERR_EXPECTING_ARRAY;
+ }
+
+ qp_add_type(packer, QP_ARRAY_OPEN);
+
+ if ((tp = qp_next(unpacker, NULL)) != QP_ARRAY2 && tp != QP_ARRAY_OPEN)
+ {
+ return ERR_EXPECTING_AT_LEAST_ONE_POINT;
+ }
+
+ for (;
+ tp == QP_ARRAY2 || tp == QP_ARRAY_OPEN;
+ (*count)++, tp = qp_next(unpacker, qp_obj))
+ {
+ qp_add_type(packer, QP_ARRAY2);
+
+ if (qp_next(unpacker, qp_obj) != QP_INT64)
+ {
+ return ERR_EXPECTING_INTEGER_TS;
+ }
+
+ if (!siridb_int64_valid_ts(siridb->time, qp_obj->via.int64))
+ {
+ return ERR_TIMESTAMP_OUT_OF_RANGE;
+ }
+
+ qp_add_int64(packer, qp_obj->via.int64);
+
+ switch (qp_next(unpacker, qp_obj))
+ {
+ case QP_RAW:
+ if (siridb_servers_check_version(siridb, "2.0.27") > 0)
+ {
+ return ERR_INCOMPATIBLE_SERVER_VERSION;
+ }
+ qp_add_raw(packer, qp_obj->via.raw, qp_obj->len);
+ break;
+
+ case QP_INT64:
+ qp_add_int64(packer, qp_obj->via.int64);
+ break;
+
+ case QP_DOUBLE:
+ qp_add_double(packer, qp_obj->via.real);
+ break;
+
+ default:
+ return ERR_UNSUPPORTED_VALUE;
+ }
+
+ if (tp == QP_ARRAY_OPEN && qp_next(unpacker, NULL) != QP_ARRAY_CLOSE)
+ break;
+ }
+
+ if (tp == QP_ARRAY_CLOSE)
+ {
+ tp = qp_next(unpacker, qp_obj);
+ }
+
+ qp_add_type(packer, QP_ARRAY_CLOSE);
+
+ return tp;
+}
+
+/*
+ * Used as uv_close_cb.
+ */
+static void INSERT_free(uv_handle_t * handle)
+{
+ siridb_insert_t * insert = (siridb_insert_t *) handle->data;
+
+ /* decrement the client reference counter */
+ sirinet_stream_decref(insert->client);
+
+ /* free insert */
+ siridb_insert_free(insert);
+
+ /* free handle */
+ free((uv_async_t *) handle);
+
+}
--- /dev/null
+/*
+ * listener.c - Contains functions for processing queries.
+ */
+#include <assert.h>
+#include <cexpr/cexpr.h>
+#include <inttypes.h>
+#include <logger/logger.h>
+#include <qpack/qpack.h>
+#include <siri/async.h>
+#include <siri/db/aggregate.h>
+#include <siri/db/group.h>
+#include <siri/db/groups.h>
+#include <siri/db/nodes.h>
+#include <siri/db/presuf.h>
+#include <siri/db/props.h>
+#include <siri/db/props.h>
+#include <siri/db/query.h>
+#include <siri/db/re.h>
+#include <siri/db/series.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/db/shard.h>
+#include <siri/db/shards.h>
+#include <siri/db/tags.h>
+#include <siri/db/user.h>
+#include <siri/db/users.h>
+#include <siri/db/listener.h>
+#include <siri/db/queries.h>
+#include <siri/db/sset.h>
+#include <siri/err.h>
+#include <siri/grammar/gramp.h>
+#include <siri/help/help.h>
+#include <siri/net/promises.h>
+#include <siri/net/protocol.h>
+#include <siri/net/clserver.h>
+#include <siri/siri.h>
+#include <xstr/xstr.h>
+#include <sys/time.h>
+
+
+static uv_async_cb SIRIDB_NODE_ENTER[CLERI_END];
+static uv_async_cb SIRIDB_NODE_EXIT[CLERI_END];
+
+uv_async_cb siridb_node_get_enter(enum cleri_grammar_ids gid)
+{
+ return SIRIDB_NODE_ENTER[gid];
+}
+
+uv_async_cb siridb_node_get_exit(enum cleri_grammar_ids gid)
+{
+ return SIRIDB_NODE_EXIT[gid];
+}
+
+
+#define MAX_ITERATE_COUNT 10000 /* ten-thousand */
+#define MAX_BATCH_REQUIRE_SHARD 100 /* after reading 100 shards, iterate */
+
+#define QP_ADD_SUCCESS qp_add_raw( \
+ query->packer, (const unsigned char *) "success_msg", 11);
+#define DEFAULT_ALLOC_COLUMNS 6
+#define IS_MASTER (query->flags & SIRIDB_QUERY_FLAG_MASTER)
+
+#define MASTER_CHECK_ONLINE(siridb) \
+if (IS_MASTER && !siridb_server_self_online(siridb->server)) \
+{ \
+ sprintf(query->err_msg, \
+ "Server '%s' is currently not available to process " \
+ "this request", \
+ siridb->server->name); \
+ siridb_query_send_error(handle, CPROTO_ERR_SERVER); \
+ return; \
+} \
+if (IS_MASTER && !siridb_pools_online(siridb)) \
+{ \
+ sprintf(query->err_msg, \
+ "At least one pool is lacking an online server to process " \
+ "this request"); \
+ siridb_query_send_error(handle, CPROTO_ERR_POOL); \
+ return; \
+}
+
+#define MASTER_CHECK_VERSION(siridb, _VERSION) \
+if (IS_MASTER) \
+{ \
+ int nservers = siridb_servers_check_version(siridb, _VERSION); \
+ if (nservers) \
+ { \
+ sprintf(query->err_msg, \
+ "At least %d server%s not running version %s " \
+ "or greater which is required for this query.", \
+ nservers, (nservers == 1) ? " is" : "s are", _VERSION); \
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY); \
+ return; \
+ } \
+}
+
+#define MASTER_CHECK_ACCESSIBLE(siridb) \
+if (IS_MASTER && !siridb_server_self_accessible(siridb->server)) \
+{ \
+ sprintf(query->err_msg, \
+ "Server '%s' is currently not accessible to process " \
+ "this request", \
+ siridb->server->name); \
+ siridb_query_send_error(handle, CPROTO_ERR_SERVER); \
+ return; \
+} \
+if (IS_MASTER && !siridb_pools_accessible(siridb)) \
+{ \
+ sprintf(query->err_msg, \
+ "At least one pool is lacking an accessible server to process " \
+ "this request"); \
+ siridb_query_send_error(handle, CPROTO_ERR_POOL); \
+ return; \
+}
+
+#define MASTER_CHECK_REINDEXING(siridb) \
+if (IS_MASTER && siridb_is_reindexing(siridb)) \
+{ \
+ sprintf(query->err_msg, \
+ "SiriDB cannot perform this request because the database is " \
+ "currently re-indexing"); \
+ siridb_query_send_error(handle, CPROTO_ERR_POOL); \
+ return; \
+}
+
+#define ON_PROMISES \
+ siri_async_decref(&handle); \
+ if (promises == NULL) \
+ { \
+ return; /* signal is raised when handle is NULL */ \
+ } \
+ if (handle == NULL) \
+ { \
+ sirinet_promises_llist_free(promises); \
+ return; /* signal is raised when handle is NULL */ \
+ }
+
+#define MEM_ERR_RET \
+ sprintf(query->err_msg, "Memory allocation error."); \
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY); \
+ return;
+
+#define FILE_ERR_RET \
+ sprintf(query->err_msg, "File error occurred."); \
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY); \
+ return;
+
+#define MSG_SUCCESS_CREATE_USER \
+ "Successfully created user '%s'."
+#define MSG_SUCCESS_DROP_USER \
+ "Successfully dropped user '%s'."
+#define MSG_SUCCESS_ALTER_USER \
+ "Successfully updated user '%s'."
+#define MSG_SUCCESS_GRANT_USER \
+ "Successfully granted permissions to user '%s'."
+#define MSG_SUCCESS_REVOKE_USER \
+ "Successfully revoked permissions from user '%s'."
+#define MSG_SUCCESS_CREATE_GROUP \
+ "Successfully created group '%s'."
+#define MSG_SUCCESS_DROP_GROUP \
+ "Successfully dropped group '%s'."
+#define MSG_SUCCESS_ALTER_GROUP \
+ "Successfully updated group '%s'."
+#define MSG_SUCCESS_ALTER_TAG \
+ "Successfully updated tag '%s'."
+#define MSG_SUCCESS_SET_DROP_THRESHOLD \
+ "Successfully changed drop_threshold from %g to %g."
+#define MSG_SUCCESS_SET_LIST_LIMIT \
+ "Successfully changed list limit from %" PRIu32 " to %" PRIu32 "."
+#define MSG_SUCCESS_SET_EXPIRATION \
+ "Successfully changed expiration from %" PRIu64 " to %" PRIu64 "."
+#define MSG_SUCCESS_SET_ADDR_PORT \
+ "Successfully changed server address to '%s'."
+#define MSG_SUCCESS_DROP_SERVER \
+ "Successfully dropped server '%s'."
+#define MSG_SUCCES_SET_LOG_LEVEL_MULTI \
+ "Successfully set log level to '%s' on %lu servers."
+#define MSG_SUCCES_SET_TEE_PIPE_NAME_MULTI \
+ "Successfully set tee_pipe name on %lu servers."
+#define MSG_SUCCES_SET_LOG_LEVEL \
+ "Successfully set log level to '%s' on '%s'."
+#define MSG_SUCCES_SET_TEE_PIPE_NAME \
+ "Successfully set tee pipe name to '%s' on '%s'."
+#define MSG_SUCCESS_SET_SELECT_POINTS_LIMIT \
+ "Successfully changed select points limit from %" PRIu32 " to %" PRIu32 "."
+#define MSG_SUCCES_DROP_SERIES \
+ "Successfully dropped %lu series."
+#define MSG_SUCCES_DROP_SHARDS \
+ "Successfully dropped %lu shards. (this number does not include " \
+ "shards which are dropped on replica servers)"
+#define MSG_SUCCES_SET_BACKUP_MODE \
+ "Successfully %s backup mode on '%s'."
+#define MSG_SUCCES_SET_TIMEZONE \
+ "Successfully changed timezone from '%s' to '%s'."
+#define MSG_ERR_SERVER_ADDRESS \
+ "Its only possible to change a servers address or port when the server " \
+ "is not connected."
+#define MSG_SUCCESS_TAG \
+ "Successfully tagged %zu series."
+#define MSG_SUCCESS_UNTAG \
+ "Successfully untagged %zu series."
+#define MSG_SUCCESS_DROP_TAG \
+ "Successfully dropped tag '%s'."
+
+static void enter_access_expr(uv_async_t * handle);
+static void enter_alter_group(uv_async_t * handle);
+static void enter_alter_series(uv_async_t * handle);
+static void enter_alter_server(uv_async_t * handle);
+static void enter_alter_servers(uv_async_t * handle);
+static void enter_alter_stmt(uv_async_t * handle);
+static void enter_alter_tag(uv_async_t * handle);
+static void enter_alter_user(uv_async_t * handle);
+static void enter_count_stmt(uv_async_t * handle);
+static void enter_create_stmt(uv_async_t * handle);
+static void enter_create_user(uv_async_t * handle);
+static void enter_drop_stmt(uv_async_t * handle);
+static void enter_grant_user(uv_async_t * handle);
+static void enter_group_tag_match(uv_async_t * handle);
+static void enter_help(uv_async_t * handle);
+static void enter_limit_expr(uv_async_t * handle);
+static void enter_list_stmt(uv_async_t * handle);
+static void enter_merge_as(uv_async_t * handle);
+static void enter_revoke_user(uv_async_t * handle);
+static void enter_select_stmt(uv_async_t * handle);
+static void enter_set_expression(uv_async_t * handle);
+static void enter_set_ignore_threshold(uv_async_t * handle);
+static void enter_set_name(uv_async_t * handle);
+static void enter_set_password(uv_async_t * handle);
+static void enter_series_all(uv_async_t * handle);
+static void enter_series_name(uv_async_t * handle);
+static void enter_series_match(uv_async_t * handle);
+static void enter_series_parentheses(uv_async_t * handle);
+static void enter_series_re(uv_async_t * handle);
+static void enter_series_setopr(uv_async_t * handle);
+static void enter_tag_series(uv_async_t * handle);
+static void enter_timeit_stmt(uv_async_t * handle);
+static void enter_untag_series(uv_async_t * handle);
+static void enter_where_xxx(uv_async_t * handle);
+static void enter_xxx_columns(uv_async_t * handle);
+
+static void exit_after_expr(uv_async_t * handle);
+static void exit_alter_group(uv_async_t * handle);
+static void exit_alter_tag(uv_async_t * handle);
+static void exit_alter_user(uv_async_t * handle);
+static void exit_before_expr(uv_async_t * handle);
+static void exit_between_expr(uv_async_t * handle);
+static void exit_calc_stmt(uv_async_t * handle);
+static void exit_count_groups(uv_async_t * handle);
+static void exit_count_pools(uv_async_t * handle);
+static void exit_count_series(uv_async_t * handle);
+static void exit_count_series_length(uv_async_t * handle);
+static void exit_count_servers(uv_async_t * handle);
+static void exit_count_servers_received(uv_async_t * handle);
+static void exit_count_servers_selected(uv_async_t * handle);
+static void exit_count_shards(uv_async_t * handle);
+static void exit_count_shards_size(uv_async_t * handle);
+static void exit_count_tags(uv_async_t * handle);
+static void exit_count_users(uv_async_t * handle);
+static void exit_create_group(uv_async_t * handle);
+static void exit_create_user(uv_async_t * handle);
+static void exit_drop_group(uv_async_t * handle);
+static void exit_drop_series(uv_async_t * handle);
+static void exit_drop_server(uv_async_t * handle);
+static void exit_drop_shards(uv_async_t * handle);
+static void exit_drop_tag(uv_async_t * handle);
+static void exit_drop_user(uv_async_t * handle);
+static void exit_grant_user(uv_async_t * handle);
+static void exit_help_xxx(uv_async_t * handle);
+static void exit_list_groups(uv_async_t * handle);
+static void exit_list_pools(uv_async_t * handle);
+static void exit_list_series(uv_async_t * handle);
+static void exit_list_servers(uv_async_t * handle);
+static void exit_list_shards(uv_async_t * handle);
+static void exit_list_tags(uv_async_t * handle);
+static void exit_list_users(uv_async_t * handle);
+static void exit_revoke_user(uv_async_t * handle);
+static void exit_select_aggregate(uv_async_t * handle);
+static void exit_select_stmt(uv_async_t * handle);
+static void exit_series_match(uv_async_t * handle);
+static void exit_series_parentheses(uv_async_t * handle);
+static void exit_set_address(uv_async_t * handle);
+static void exit_set_backup_mode(uv_async_t * handle);
+static void exit_set_drop_threshold(uv_async_t * handle);
+static void exit_set_expiration_log(uv_async_t * handle);
+static void exit_set_expiration_num(uv_async_t * handle);
+static void exit_set_list_limit(uv_async_t * handle);
+static void exit_set_log_level(uv_async_t * handle);
+static void exit_set_port(uv_async_t * handle);
+static void exit_set_select_points_limit(uv_async_t * handle);
+static void exit_set_tee_pipe_name(uv_async_t * handle);
+static void exit_set_timezone(uv_async_t * handle);
+static void exit_show_stmt(uv_async_t * handle);
+static void exit_tag_series(uv_async_t * handle);
+static void exit_timeit_stmt(uv_async_t * handle);
+static void exit_untag_series(uv_async_t * handle);
+
+/* async loop functions */
+static void async_count_series(uv_async_t * handle);
+static void async_count_series_length(uv_async_t * handle);
+static void async_drop_series(uv_async_t * handle);
+static void async_drop_shards(uv_async_t * handle);
+static void async_filter_series(uv_async_t * handle);
+static void async_list_series(uv_async_t * handle);
+static void async_no_points_aggregate(uv_async_t * handle);
+static void async_select_aggregate(uv_async_t * handle);
+static void async_series_re(uv_async_t * handle);
+
+/* on response functions */
+static void on_ack_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+static void on_alter_xxx_response(vec_t * promises, uv_async_t * handle);
+static void on_count_xxx_response(vec_t * promises, uv_async_t * handle);
+static void on_drop_series_response(vec_t * promises, uv_async_t * handle);
+static void on_drop_shards_response(vec_t * promises, uv_async_t * handle);
+static void on_groups_response(vec_t * promises, uv_async_t * handle);
+static void on_tags_response(vec_t * promises, uv_async_t * handle);
+static void on_list_xxx_response(vec_t * promises, uv_async_t * handle);
+static void on_select_response(vec_t * promises, uv_async_t * handle);
+static void on_update_xxx_response(vec_t * promises, uv_async_t * handle);
+static void on_tag_response(vec_t * promises, uv_async_t * handle);
+
+/* helper functions */
+static void master_select_work(uv_work_t * handle);
+static void master_select_work_finish(uv_work_t * work, int status);
+static int items_select_master(
+ const char * name,
+ size_t len,
+ siridb_points_t * points,
+ uv_async_t * handle);
+static int items_select_master_merge(
+ const char * name,
+ size_t len,
+ vec_t * plist,
+ uv_async_t * handle);
+static int items_select_other(
+ const char * name,
+ size_t len,
+ siridb_points_t * points,
+ uv_async_t * handle);
+static int items_select_other_merge(
+ const char * name,
+ size_t len,
+ vec_t * plist,
+ uv_async_t * handle);
+static void on_select_unpack_points(
+ qp_unpacker_t * unpacker,
+ query_select_t * q_select,
+ qp_obj_t * qp_name,
+ qp_obj_t * qp_tp,
+ qp_obj_t * qp_len,
+ qp_obj_t * qp_points,
+ uint32_t select_points_limit);
+static void on_select_unpack_merged_points(
+ qp_unpacker_t * unpacker,
+ query_select_t * q_select,
+ qp_obj_t * qp_name,
+ qp_obj_t * qp_tp,
+ qp_obj_t * qp_len,
+ qp_obj_t * qp_points,
+ uint32_t select_points_limit);
+
+static int values_list_groups(siridb_group_t * group, uv_async_t * handle);
+static int values_count_groups(siridb_group_t * group, uv_async_t * handle);
+static void finish_list_groups(uv_async_t * handle);
+static void finish_count_groups(uv_async_t * handle);
+static int values_list_tags(siridb_tag_t * tag, uv_async_t * handle);
+static int values_count_tags(siridb_tag_t * tag, uv_async_t * handle);
+static void finish_list_tags(uv_async_t * handle);
+static void finish_count_tags(uv_async_t * handle);
+
+
+/* address bindings for default list properties */
+static uint32_t GID_K_NAME = CLERI_GID_K_NAME;
+static uint32_t GID_K_POOL = CLERI_GID_K_POOL;
+static uint32_t GID_K_VERSION = CLERI_GID_K_VERSION;
+static uint32_t GID_K_ONLINE = CLERI_GID_K_ONLINE;
+static uint32_t GID_K_STATUS = CLERI_GID_K_STATUS;
+static uint32_t GID_K_SERVER = CLERI_GID_K_SERVER;
+static uint32_t GID_K_SERVERS = CLERI_GID_K_SERVERS;
+static uint32_t GID_K_SERIES = CLERI_GID_K_SERIES;
+static uint32_t GID_K_SID = CLERI_GID_K_SID;
+static uint32_t GID_K_START = CLERI_GID_K_START;
+static uint32_t GID_K_END = CLERI_GID_K_END;
+
+/*
+ * Start SIRIPARSER_ASYNC_NEXT_NODE
+ */
+#define SIRIPARSER_ASYNC_NEXT_NODE \
+siridb_nodes_next(&query->nodes); \
+if (query->nodes == NULL) \
+{ \
+ siridb_send_query_result(handle); \
+} \
+else \
+{ \
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free); \
+ handle = malloc(sizeof(uv_async_t)); \
+ if (handle == NULL) \
+ { \
+ ERR_ALLOC \
+ } \
+ else \
+ { \
+ handle->data = query; \
+ uv_async_init(siri.loop, handle, (uv_async_cb) query->nodes->cb); \
+ uv_async_send(handle); \
+ } \
+}
+
+#define SIRIPARSER_NEXT_NODE \
+siridb_nodes_next(&query->nodes); \
+if (query->nodes == NULL) \
+{ \
+ siridb_send_query_result(handle); \
+} \
+else \
+{ \
+ handle->data = query; \
+ query->nodes->cb(handle); \
+}
+
+
+/*
+ * Start SIRIPARSER_MASTER_CHECK_ACCESS
+ */
+#define SIRIPARSER_MASTER_CHECK_ACCESS(user, ACCESS_BIT) \
+if (IS_MASTER && \
+ !siridb_user_check_access( \
+ user, \
+ ACCESS_BIT, \
+ query->err_msg)) \
+{ \
+ siridb_query_send_error(handle, CPROTO_ERR_USER_ACCESS); \
+ return; \
+}
+
+void siridb_init_listener(void)
+{
+ uint_fast16_t i;
+
+ for (i = 0; i < CLERI_END; i++)
+ {
+ SIRIDB_NODE_ENTER[i] = NULL;
+ SIRIDB_NODE_EXIT[i] = NULL;
+ }
+
+ SIRIDB_NODE_ENTER[CLERI_GID_ACCESS_EXPR] = enter_access_expr;
+ SIRIDB_NODE_ENTER[CLERI_GID_ALTER_GROUP] = enter_alter_group;
+ SIRIDB_NODE_ENTER[CLERI_GID_ALTER_SERIES] = enter_alter_series;
+ SIRIDB_NODE_ENTER[CLERI_GID_ALTER_SERVER] = enter_alter_server;
+ SIRIDB_NODE_ENTER[CLERI_GID_ALTER_SERVERS] = enter_alter_servers;
+ SIRIDB_NODE_ENTER[CLERI_GID_ALTER_STMT] = enter_alter_stmt;
+ SIRIDB_NODE_ENTER[CLERI_GID_ALTER_TAG] = enter_alter_tag;
+ SIRIDB_NODE_ENTER[CLERI_GID_ALTER_USER] = enter_alter_user;
+ SIRIDB_NODE_ENTER[CLERI_GID_COUNT_STMT] = enter_count_stmt;
+ SIRIDB_NODE_ENTER[CLERI_GID_CREATE_STMT] = enter_create_stmt;
+ SIRIDB_NODE_ENTER[CLERI_GID_CREATE_USER] = enter_create_user;
+ SIRIDB_NODE_ENTER[CLERI_GID_DROP_STMT] = enter_drop_stmt;
+ SIRIDB_NODE_ENTER[CLERI_GID_GRANT_USER] = enter_grant_user;
+ SIRIDB_NODE_ENTER[CLERI_GID_GROUP_COLUMNS] = enter_xxx_columns;
+ SIRIDB_NODE_ENTER[CLERI_GID_GROUP_TAG_MATCH] = enter_group_tag_match;
+ SIRIDB_NODE_ENTER[CLERI_GID_HELP_STMT] = enter_help;
+ SIRIDB_NODE_ENTER[CLERI_GID_LIMIT_EXPR] = enter_limit_expr;
+ SIRIDB_NODE_ENTER[CLERI_GID_LIST_STMT] = enter_list_stmt;
+ SIRIDB_NODE_ENTER[CLERI_GID_MERGE_AS] = enter_merge_as;
+ SIRIDB_NODE_ENTER[CLERI_GID_POOL_COLUMNS] = enter_xxx_columns;
+ SIRIDB_NODE_ENTER[CLERI_GID_REVOKE_USER] = enter_revoke_user;
+ SIRIDB_NODE_ENTER[CLERI_GID_SELECT_STMT] = enter_select_stmt;
+ SIRIDB_NODE_ENTER[CLERI_GID_SET_EXPRESSION] = enter_set_expression;
+ SIRIDB_NODE_ENTER[CLERI_GID_SET_IGNORE_THRESHOLD] = enter_set_ignore_threshold;
+ SIRIDB_NODE_ENTER[CLERI_GID_SET_NAME] = enter_set_name;
+ SIRIDB_NODE_ENTER[CLERI_GID_SET_PASSWORD] = enter_set_password;
+ SIRIDB_NODE_ENTER[CLERI_GID_SERIES_COLUMNS] = enter_xxx_columns;
+ SIRIDB_NODE_ENTER[CLERI_GID_SERVER_COLUMNS] = enter_xxx_columns;
+ SIRIDB_NODE_ENTER[CLERI_GID_SERIES_ALL] = enter_series_all;
+ SIRIDB_NODE_ENTER[CLERI_GID_SERIES_NAME] = enter_series_name;
+ SIRIDB_NODE_ENTER[CLERI_GID_SERIES_MATCH] = enter_series_match;
+ SIRIDB_NODE_ENTER[CLERI_GID_SERIES_PARENTHESES] = enter_series_parentheses;
+ SIRIDB_NODE_ENTER[CLERI_GID_SERIES_RE] = enter_series_re;
+ SIRIDB_NODE_ENTER[CLERI_GID_SERIES_SETOPR] = enter_series_setopr;
+ SIRIDB_NODE_ENTER[CLERI_GID_SHARD_COLUMNS] = enter_xxx_columns;
+ SIRIDB_NODE_ENTER[CLERI_GID_TAG_COLUMNS] = enter_xxx_columns;
+ SIRIDB_NODE_ENTER[CLERI_GID_TAG_SERIES] = enter_tag_series;
+ SIRIDB_NODE_ENTER[CLERI_GID_TIMEIT_STMT] = enter_timeit_stmt;
+ SIRIDB_NODE_ENTER[CLERI_GID_UNTAG_SERIES] = enter_untag_series;
+ SIRIDB_NODE_ENTER[CLERI_GID_USER_COLUMNS] = enter_xxx_columns;
+ SIRIDB_NODE_ENTER[CLERI_GID_WHERE_GROUP] = enter_where_xxx;
+ SIRIDB_NODE_ENTER[CLERI_GID_WHERE_POOL] = enter_where_xxx;
+ SIRIDB_NODE_ENTER[CLERI_GID_WHERE_SERIES] = enter_where_xxx;
+ SIRIDB_NODE_ENTER[CLERI_GID_WHERE_SERVER] = enter_where_xxx;
+ SIRIDB_NODE_ENTER[CLERI_GID_WHERE_SHARD] = enter_where_xxx;
+ SIRIDB_NODE_ENTER[CLERI_GID_WHERE_TAG] = enter_where_xxx;
+ SIRIDB_NODE_ENTER[CLERI_GID_WHERE_USER] = enter_where_xxx;
+
+
+ SIRIDB_NODE_EXIT[CLERI_GID_AFTER_EXPR] = exit_after_expr;
+ SIRIDB_NODE_EXIT[CLERI_GID_ALTER_GROUP] = exit_alter_group;
+ SIRIDB_NODE_EXIT[CLERI_GID_ALTER_TAG] = exit_alter_tag;
+ SIRIDB_NODE_EXIT[CLERI_GID_ALTER_USER] = exit_alter_user;
+ SIRIDB_NODE_EXIT[CLERI_GID_BEFORE_EXPR] = exit_before_expr;
+ SIRIDB_NODE_EXIT[CLERI_GID_BETWEEN_EXPR] = exit_between_expr;
+ SIRIDB_NODE_EXIT[CLERI_GID_CALC_STMT] = exit_calc_stmt;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_GROUPS] = exit_count_groups;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_POOLS] = exit_count_pools;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_SERIES] = exit_count_series;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_SERIES_LENGTH] = exit_count_series_length;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_SERVERS] = exit_count_servers;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_SERVERS_RECEIVED] = exit_count_servers_received;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_SERVERS_SELECTED] = exit_count_servers_selected;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_SHARDS] = exit_count_shards;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_SHARDS_SIZE] = exit_count_shards_size;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_TAGS] = exit_count_tags;
+ SIRIDB_NODE_EXIT[CLERI_GID_COUNT_USERS] = exit_count_users;
+ SIRIDB_NODE_EXIT[CLERI_GID_CREATE_GROUP] = exit_create_group;
+ SIRIDB_NODE_EXIT[CLERI_GID_CREATE_USER] = exit_create_user;
+ SIRIDB_NODE_EXIT[CLERI_GID_DROP_GROUP] = exit_drop_group;
+ SIRIDB_NODE_EXIT[CLERI_GID_DROP_SERIES] = exit_drop_series;
+ SIRIDB_NODE_EXIT[CLERI_GID_DROP_SERVER] = exit_drop_server;
+ SIRIDB_NODE_EXIT[CLERI_GID_DROP_SHARDS] = exit_drop_shards;
+ SIRIDB_NODE_EXIT[CLERI_GID_DROP_TAG] = exit_drop_tag;
+ SIRIDB_NODE_EXIT[CLERI_GID_DROP_USER] = exit_drop_user;
+ SIRIDB_NODE_EXIT[CLERI_GID_GRANT_USER] = exit_grant_user;
+ SIRIDB_NODE_EXIT[CLERI_GID_LIST_GROUPS] = exit_list_groups;
+ SIRIDB_NODE_EXIT[CLERI_GID_LIST_POOLS] = exit_list_pools;
+ SIRIDB_NODE_EXIT[CLERI_GID_LIST_SERIES] = exit_list_series;
+ SIRIDB_NODE_EXIT[CLERI_GID_LIST_SERVERS] = exit_list_servers;
+ SIRIDB_NODE_EXIT[CLERI_GID_LIST_SHARDS] = exit_list_shards;
+ SIRIDB_NODE_EXIT[CLERI_GID_LIST_TAGS] = exit_list_tags;
+ SIRIDB_NODE_EXIT[CLERI_GID_LIST_USERS] = exit_list_users;
+ SIRIDB_NODE_EXIT[CLERI_GID_REVOKE_USER] = exit_revoke_user;
+ SIRIDB_NODE_EXIT[CLERI_GID_SELECT_AGGREGATE] = exit_select_aggregate;
+ SIRIDB_NODE_EXIT[CLERI_GID_SELECT_STMT] = exit_select_stmt;
+ SIRIDB_NODE_EXIT[CLERI_GID_SERIES_MATCH] = exit_series_match;
+ SIRIDB_NODE_EXIT[CLERI_GID_SERIES_PARENTHESES] = exit_series_parentheses;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_ADDRESS] = exit_set_address;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_BACKUP_MODE] = exit_set_backup_mode;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_DROP_THRESHOLD] = exit_set_drop_threshold;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_EXPIRATION_LOG] = exit_set_expiration_log;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_EXPIRATION_NUM] = exit_set_expiration_num;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_LIST_LIMIT] = exit_set_list_limit;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_LOG_LEVEL] = exit_set_log_level;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_PORT] = exit_set_port;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_SELECT_POINTS_LIMIT] = exit_set_select_points_limit;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_TEE_PIPE_NAME] = exit_set_tee_pipe_name;
+ SIRIDB_NODE_EXIT[CLERI_GID_SET_TIMEZONE] = exit_set_timezone;
+ SIRIDB_NODE_EXIT[CLERI_GID_SHOW_STMT] = exit_show_stmt;
+ SIRIDB_NODE_EXIT[CLERI_GID_TAG_SERIES] = exit_tag_series;
+ SIRIDB_NODE_EXIT[CLERI_GID_TIMEIT_STMT] = exit_timeit_stmt;
+ SIRIDB_NODE_EXIT[CLERI_GID_UNTAG_SERIES] = exit_untag_series;
+
+ for (i = HELP_OFFSET; i < HELP_OFFSET + HELP_COUNT; i++)
+ {
+ SIRIDB_NODE_EXIT[i] = exit_help_xxx;
+ }
+}
+
+/******************************************************************************
+ * Enter functions
+ *****************************************************************************/
+
+static void enter_access_expr(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ /* bind ACCESS_EXPR children to query */
+ query->data = query->nodes->node->children;
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_alter_group(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * group_node =
+ query->nodes->node->children->next->node;
+ siridb_group_t * group;
+
+ char name[group_node->len - 1];
+ xstr_extract_string(name, group_node->str, group_node->len);
+
+ if ((group = ct_get(siridb->groups->groups, name)) == NULL)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find group: '%s'",
+ name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ q_alter->alter_tp = QUERY_ALTER_GROUP;
+ q_alter->via.group = group;
+ siridb_group_incref(group);
+
+ SIRIPARSER_NEXT_NODE
+ }
+}
+
+static void enter_alter_tag(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * tag_node = query->nodes->node->children->next->node;
+ siridb_tag_t * tag;
+
+ char name[tag_node->len - 1];
+ xstr_extract_string(name, tag_node->str, tag_node->len);
+
+ if ((tag = ct_get(siridb->tags->tags, name)) == NULL)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find tag: '%s'",
+ name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ q_alter->alter_tp = QUERY_ALTER_TAG;
+ q_alter->via.tag = tag;
+ siridb_tag_incref(tag);
+
+ SIRIPARSER_NEXT_NODE
+ }
+}
+
+static void enter_alter_series(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ q_alter->alter_tp = QUERY_ALTER_SERIES;
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_alter_server(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+ siridb_server_t * server = siridb_server_from_node(
+ siridb,
+ query->nodes->node->children->next->node->children->node,
+ query->err_msg);
+
+ if (server == NULL)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ q_alter->alter_tp = QUERY_ALTER_SERVER;
+ q_alter->via.server = server;
+ siridb_server_incref(server);
+
+ SIRIPARSER_NEXT_NODE
+ }
+}
+
+static void enter_alter_servers(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ ((query_alter_t *) query->data)->alter_tp = QUERY_ALTER_SERVERS;
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_alter_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_user_t * db_user = query->client->origin;
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_ALTER)
+
+ assert (query->packer == NULL);
+
+ query->packer = sirinet_packer_new(1024);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ query->data = query_alter_new();
+
+ if (query->data == NULL)
+ {
+ MEM_ERR_RET
+ }
+ query->free_cb = (uv_close_cb) query_alter_free;
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_alter_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * user_node =
+ query->nodes->node->children->next->node;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+ siridb_user_t * user;
+
+ char name[user_node->len - 1];
+ xstr_extract_string(name, user_node->str, user_node->len);
+
+ if ((user = siridb_users_get_user(siridb, name, NULL)) == NULL)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find user: '%s'",
+ name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ q_alter->alter_tp = QUERY_ALTER_USER;
+ q_alter->via.user = user;
+ siridb_user_incref(user);
+
+ SIRIPARSER_NEXT_NODE
+ }
+}
+
+static void enter_count_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_user_t * db_user = query->client->origin;
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_COUNT)
+
+ assert (query->packer == NULL);
+
+ query->packer = sirinet_packer_new(256);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ query->data = query_count_new();
+
+ if (query->data == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ query->free_cb = (uv_close_cb) query_count_free;
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_create_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_user_t * db_user = query->client->origin;
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_CREATE)
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_create_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ /* bind user object to data and set correct free call */
+ query_alter_t * q_alter = query_alter_new();
+
+ query->data = q_alter;
+
+ if (q_alter == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ query->free_cb = (uv_close_cb) query_alter_free;
+ q_alter->via.user = siridb_user_new();
+
+ if (q_alter->via.user == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ q_alter->alter_tp = QUERY_ALTER_USER;
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_drop_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_user_t * db_user = query->client->origin;
+
+ assert (query->packer == NULL);
+
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_DROP)
+
+ query->packer = sirinet_packer_new(1024);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ query->data = query_drop_new();
+
+ if (query->data == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ query->free_cb = (uv_close_cb) query_drop_free;
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_grant_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_user_t * db_user = query->client->origin;
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_GRANT)
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * user_node =
+ query->nodes->node->children->next->node;
+ siridb_user_t * user;
+ char username[user_node->len - 1];
+ xstr_extract_string(username, user_node->str, user_node->len);
+
+ if ((user = siridb_users_get_user(siridb, username, NULL)) == NULL)
+ {
+ snprintf(query->err_msg, SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find user: '%s'", username);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ user->access_bit |=
+ siridb_access_from_children((cleri_children_t *) query->data);
+
+ query_alter_t * q_alter = query->data = query_alter_new();
+ if (q_alter == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ siridb_user_incref(user);
+
+ query->free_cb = (uv_close_cb) query_alter_free;
+ q_alter->alter_tp = QUERY_ALTER_USER;
+ q_alter->via.user = user;
+
+ SIRIPARSER_NEXT_NODE
+ }
+}
+static void enter_group_tag_match(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ cleri_node_t * node = query->nodes->node;
+ query_wrapper_t * q_wrapper = query->data;
+ siridb_group_t * group;
+ siridb_tag_t * tag = NULL;
+
+
+ /* we must send this query to all pools */
+ if (q_wrapper->pmap != NULL)
+ {
+ imap_free(q_wrapper->pmap, NULL);
+ q_wrapper->pmap = NULL;
+ }
+
+ char group_or_tag_name[node->len - 1];
+
+ /* extract series name */
+ xstr_extract_string(group_or_tag_name, node->str, node->len);
+
+ if ((group = ct_get(siridb->groups->groups, group_or_tag_name)) == NULL &&
+ (tag = ct_get(siridb->tags->tags, group_or_tag_name)) == NULL)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find group or tag '%s'",
+ group_or_tag_name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ siridb_series_t * series;
+ size_t i;
+
+ q_wrapper->series_tmp = (q_wrapper->update_cb == NULL) ?
+ q_wrapper->series_map : imap_new();
+
+ if (q_wrapper->series_tmp == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ if (group)
+ {
+ uv_mutex_lock(&siridb->groups->mutex);
+
+ for (i = 0; i < group->series->len; i++)
+ {
+ series = (siridb_series_t *) group->series->data[i];
+ siridb_series_incref(series);
+ if (imap_add(q_wrapper->series_tmp, series->id, series))
+ {
+ log_critical("Cannot add series to temporary map.");
+ siridb_series_decref(series);
+ }
+ }
+
+ uv_mutex_unlock(&siridb->groups->mutex);
+ }
+ else if (tag) /* check to prevent compile warning */
+ {
+ vec_t * tag_series;
+
+ assert (tag != NULL);
+
+ uv_mutex_lock(&siridb->tags->mutex);
+
+ tag_series = imap_vec(tag->series);
+
+ if (tag_series != NULL)
+ {
+ for (i = 0; i < tag_series->len; i++)
+ {
+ series = (siridb_series_t *) tag_series->data[i];
+ siridb_series_incref(series);
+ if (imap_add(q_wrapper->series_tmp, series->id, series))
+ {
+ log_critical("Cannot add series to temporary map.");
+ siridb_series_decref(series);
+ }
+ }
+ }
+
+ uv_mutex_unlock(&siridb->tags->mutex);
+ }
+
+ if (q_wrapper->update_cb != NULL)
+ {
+ (*q_wrapper->update_cb)(
+ q_wrapper->series_map,
+ q_wrapper->series_tmp,
+ (imap_free_cb) &siridb__series_decref);
+ }
+
+ q_wrapper->series_tmp = NULL;
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void enter_help(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ cleri_node_t * node = query->nodes->node;
+
+ query->data = strndup(node->str, node->len);
+
+ if (query->data == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ query->free_cb = (uv_close_cb) query_help_free;
+
+ xstr_split_join(query->data, ' ', '_');
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void enter_limit_expr(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_list_t * qlist = (query_list_t *) query->data;
+ int64_t limit = CLERI_NODE_DATA(query->nodes->node->children->next->node);
+
+ if (limit <= 0 || limit > siridb->list_limit)
+ {
+ snprintf(query->err_msg, SIRIDB_MAX_SIZE_ERR_MSG,
+ "Limit must be a value between 1 and %" PRIu32
+ " but received: %" PRId64
+ " (optionally the limit can be changed, "
+ "see 'help alter database')",
+ siridb->list_limit,
+ limit);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ qlist->limit = limit;
+ SIRIPARSER_NEXT_NODE
+ }
+}
+
+static void enter_list_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_user_t * db_user = query->client->origin;
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_LIST)
+
+ assert (query->packer == NULL);
+
+ query->packer = sirinet_packer_new(QP_SUGGESTED_SIZE);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ qp_add_raw(query->packer, (const unsigned char *) "columns", 7);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ query->data = query_list_new();
+
+ if (query->data == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ query->free_cb = (uv_close_cb) query_list_free;
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_merge_as(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_select_t * q_select = query->data;
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+ q_select->merge_as = malloc(node->len - 1);
+
+ if (q_select->merge_as == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ xstr_extract_string(q_select->merge_as, node->str, node->len);
+
+ if (IS_MASTER && query->nodes->node->children->next->next->next != NULL)
+ {
+ q_select->mlist = siridb_aggregate_list(
+ query->nodes->node->children->next->next->next->node->
+ children->node->children->next->node->children,
+ query->err_msg);
+
+ if (q_select->mlist == NULL)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void enter_revoke_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_user_t * db_user = query->client->origin;
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_REVOKE)
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * user_node =
+ query->nodes->node->children->next->node;
+ siridb_user_t * user;
+ char username[user_node->len - 1];
+ xstr_extract_string(username, user_node->str, user_node->len);
+
+ if ((user = siridb_users_get_user(siridb, username, NULL)) == NULL)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find user: '%s'",
+ username);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ user->access_bit &=
+ ~siridb_access_from_children((cleri_children_t *) query->data);
+
+ query_alter_t * q_alter = query->data = query_alter_new();
+
+ if (q_alter == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ siridb_user_incref(user);
+
+ query->free_cb = (uv_close_cb) query_alter_free;
+ q_alter->alter_tp = QUERY_ALTER_USER;
+ q_alter->via.user = user;
+
+ SIRIPARSER_NEXT_NODE
+ }
+}
+
+static void enter_select_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_select_t * q_select;
+ cleri_children_t * child;
+ int skip_get_points;
+ siridb_user_t * db_user = query->client->origin;
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_SELECT)
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ assert (query->packer == NULL && query->data == NULL);
+
+ query->data = q_select = query_select_new();
+
+ if (q_select == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ /* this is not critical since pmap is allowed to be NULL */
+ ((query_select_t *) query->data)->pmap =
+ (!IS_MASTER || siridb_is_reindexing(siridb)) ?
+ NULL : imap_new();
+
+ /* child is always the ',' and child->next the node */
+ child = query->nodes->node->children->next->node->children;
+ skip_get_points = siridb_aggregate_can_skip(child);
+
+ child = child->next;
+ while (child != NULL)
+ {
+ if (skip_get_points && !siridb_aggregate_can_skip(child->next))
+ {
+ skip_get_points = 0;
+ }
+ q_select->nselects++;
+ child = child->next->next;
+ }
+
+ if (skip_get_points)
+ {
+ q_select->flags |= QUERIES_SKIP_GET_POINTS;
+ }
+
+ query->free_cb = (uv_close_cb) query_select_free;
+ query->packer = sirinet_packer_new(QP_SUGGESTED_SIZE);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void enter_set_expression(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+
+ if (siridb_group_update_expression(
+ siridb->groups,
+ q_alter->via.group,
+ node->str,
+ node->len,
+ query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void enter_set_ignore_threshold(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_wrapper_t * q_wrapper = (query_wrapper_t *) query->data;
+
+ if ( query->nodes->node->children->next->next->node->children->node->
+ cl_obj->gid == CLERI_GID_K_TRUE)
+ {
+ q_wrapper->flags |= QUERIES_IGNORE_DROP_THRESHOLD;
+ }
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_set_name(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ cleri_node_t * name_node =
+ query->nodes->node->children->next->next->node;
+
+ char name[name_node->len - 1];
+ xstr_extract_string(name, name_node->str, name_node->len);
+
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+ switch (q_alter->alter_tp)
+ {
+ case QUERY_ALTER_USER:
+ if (siridb_user_set_name(
+ siridb,
+ q_alter->via.user,
+ name,
+ query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ break;
+ case QUERY_ALTER_GROUP:
+ if (siridb_group_set_name(
+ siridb,
+ q_alter->via.group,
+ name,
+ query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ break;
+ case QUERY_ALTER_TAG:
+ if (siridb_tag_set_name(
+ siridb,
+ q_alter->via.tag,
+ name,
+ query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ break;
+ case QUERY_ALTER_NONE:
+ case QUERY_ALTER_DATABASE:
+ case QUERY_ALTER_SERVER:
+ default:
+ assert (0);
+ }
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_set_password(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_user_t * user = ((query_alter_t *) query->data)->via.user;
+
+ cleri_node_t * pw_node =
+ query->nodes->node->children->next->next->node;
+
+ char password[pw_node->len - 1];
+ xstr_extract_string(password, pw_node->str, pw_node->len);
+
+ if (siridb_user_set_password(user, password, query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ SIRIPARSER_NEXT_NODE
+ }
+}
+
+static void enter_series_name(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ cleri_node_t * node = query->nodes->node;
+ siridb_t * siridb = query->client->siridb;
+ query_wrapper_t * q_wrapper = query->data;
+ siridb_series_t * series = NULL;
+ uint16_t pool;
+ char series_name[node->len - 1];
+
+ /* extract series name */
+ xstr_extract_string(series_name, node->str, node->len);
+
+ if (siridb_is_reindexing(siridb))
+ {
+ series = (siridb_series_t *) ct_get(siridb->series, series_name);
+ }
+ else
+ {
+ /* get pool for series name */
+ pool = siridb_lookup_sn(siridb->pools->lookup, series_name);
+
+ /* check if this series belongs to 'this' pool and if so get the series */
+ if (pool == siridb->server->pool)
+ {
+ series = (siridb_series_t *) ct_get(siridb->series, series_name);
+#ifdef SERIESMUSTEXIST
+ if (series == NULL)
+ {
+ /* the series does not exist */
+ snprintf(query->err_msg, SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find series: '%s'", series_name);
+
+ /* free series_name and return with send_errror.. */
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+#endif
+ }
+ else if (q_wrapper->pmap != NULL && imap_set(
+ q_wrapper->pmap,
+ pool,
+ (siridb_pool_t *) (siridb->pools->pool + pool)) < 0)
+ {
+ log_critical("Cannot add pool to pool map.");
+ }
+ }
+
+ if (series == NULL)
+ {
+ if (q_wrapper->update_cb == &imap_intersection_ref)
+ {
+ imap_free(
+ q_wrapper->series_map,
+ (imap_free_cb) &siridb__series_decref);
+
+ q_wrapper->series_map = imap_new();
+
+ if (q_wrapper->series_map == NULL)
+ {
+ MEM_ERR_RET
+ }
+ }
+ }
+ else
+ {
+ if ( q_wrapper->update_cb == NULL ||
+ q_wrapper->update_cb == &imap_union_ref)
+ {
+ if (imap_set(q_wrapper->series_map, series->id, series) == 1)
+ {
+ siridb_series_incref(series);
+ }
+ }
+ else if (q_wrapper->update_cb == &imap_difference_ref)
+ {
+ series = (siridb_series_t *) imap_pop(
+ q_wrapper->series_map,
+ series->id);
+ if (series != NULL)
+ {
+ siridb_series_decref(series);
+ }
+ }
+ else if (q_wrapper->update_cb == &imap_intersection_ref)
+ {
+ series = (siridb_series_t *) imap_get(
+ q_wrapper->series_map,
+ series->id);
+
+ if (series != NULL)
+ {
+ siridb_series_incref(series);
+ }
+
+ imap_free(
+ q_wrapper->series_map,
+ (imap_free_cb) &siridb__series_decref);
+
+ q_wrapper->series_map = imap_new();
+
+ if (q_wrapper->series_map == NULL)
+ {
+ if (series != NULL)
+ {
+ siridb_series_decref(series);
+ }
+ MEM_ERR_RET
+ }
+
+ if (series != NULL)
+ {
+ if (imap_set(q_wrapper->series_map, series->id, series) != 1)
+ {
+ siridb_series_decref(series);
+ MEM_ERR_RET
+ }
+ }
+ }
+ else if (q_wrapper->update_cb == &imap_symmetric_difference_ref)
+ {
+ switch (imap_set(q_wrapper->series_map, series->id, series))
+ {
+ case 0:
+ series = (siridb_series_t *) imap_pop(
+ q_wrapper->series_map,
+ series->id);
+ siridb_series_decref(series);
+ break;
+
+ case 1:
+ siridb_series_incref(series);
+ break;
+
+ default:
+ MEM_ERR_RET
+ }
+ }
+ else
+ {
+ /* we should not get here */
+ assert (0);
+ }
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void enter_series_match(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_wrapper_t * q_wrapper = query->data;
+
+ if ((q_wrapper->series_map = imap_new()) == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_series_parentheses(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_wrapper_t * q_wrapper = query->data;
+ siridb_sset_t * sset;
+
+ assert (q_wrapper->series_map != NULL);
+
+ if (q_wrapper->sset_vec == NULL)
+ {
+ if ((q_wrapper->sset_vec = vec_new(1)) == NULL)
+ {
+ MEM_ERR_RET
+ }
+ }
+
+ if (q_wrapper->update_cb != NULL)
+ {
+ sset = siridb_sset_new(q_wrapper->series_map, q_wrapper->update_cb);
+ if (sset == NULL ||
+ vec_append_safe(&q_wrapper->sset_vec, sset) ||
+ (q_wrapper->series_map = imap_new()) == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ q_wrapper->update_cb = NULL;
+ }
+ else
+ {
+ sset = siridb_sset_new(NULL, NULL);
+ if (sset == NULL || vec_append_safe(&q_wrapper->sset_vec, sset))
+ {
+ MEM_ERR_RET
+ }
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void enter_series_all(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_series_t * series;
+ query_wrapper_t * q_wrapper = query->data;
+
+ /* we must send this query to all pools */
+ if (q_wrapper->pmap != NULL)
+ {
+ imap_free(q_wrapper->pmap, NULL);
+ q_wrapper->pmap = NULL;
+ }
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ q_wrapper->vec = imap_2vec_ref(
+ ( q_wrapper->update_cb == NULL ||
+ q_wrapper->update_cb == &imap_union_ref ||
+ q_wrapper->update_cb == &imap_symmetric_difference_ref) ?
+ siridb->series_map : q_wrapper->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ q_wrapper->series_tmp = (q_wrapper->update_cb == NULL) ?
+ q_wrapper->series_map : imap_new();
+
+ if (q_wrapper->vec == NULL || q_wrapper->series_tmp == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ for (q_wrapper->vec_index = 0;
+ q_wrapper->vec_index < q_wrapper->vec->len;
+ ++q_wrapper->vec_index)
+ {
+ series = q_wrapper->vec->data[q_wrapper->vec_index];
+ if (imap_add(q_wrapper->series_tmp, series->id, series))
+ {
+ MEM_ERR_RET
+ }
+ }
+
+ vec_free(q_wrapper->vec);
+ q_wrapper->vec = NULL;
+ q_wrapper->vec_index = 0;
+
+ if (q_wrapper->update_cb != NULL)
+ {
+ (*q_wrapper->update_cb)(
+ q_wrapper->series_map,
+ q_wrapper->series_tmp,
+ (imap_free_cb) &siridb__series_decref);
+ }
+
+ q_wrapper->series_tmp = NULL;
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void enter_series_re(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ cleri_node_t * node = query->nodes->node;
+ query_wrapper_t * q_wrapper = query->data;
+
+ /* we must send this query to all pools */
+ if (q_wrapper->pmap != NULL)
+ {
+ imap_free(q_wrapper->pmap, NULL);
+ q_wrapper->pmap = NULL;
+ }
+
+ /* extract and compile regular expression */
+ if (siridb_re_compile(
+ &q_wrapper->regex,
+ &q_wrapper->match_data,
+ node->str,
+ node->len,
+ query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ uv_mutex_lock(&siridb->series_mutex);
+
+ q_wrapper->vec = imap_2vec_ref(
+ ( q_wrapper->update_cb == NULL ||
+ q_wrapper->update_cb == &imap_union_ref ||
+ q_wrapper->update_cb == &imap_symmetric_difference_ref) ?
+ siridb->series_map : q_wrapper->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ q_wrapper->series_tmp = (q_wrapper->update_cb == NULL) ?
+ q_wrapper->series_map : imap_new();
+
+ if (q_wrapper->vec == NULL || q_wrapper->series_tmp == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ uv_async_t * next = malloc(sizeof(uv_async_t));
+
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) async_series_re);
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+ }
+
+ /* handle is handled or a signal is raised */
+}
+
+static void enter_series_setopr(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_wrapper_t * q_wrapper = query->data;
+
+ switch (query->nodes->node->children->node->cl_obj->gid)
+ {
+ case CLERI_GID_K_UNION:
+ q_wrapper->update_cb = &imap_union_ref;
+ break;
+ case CLERI_GID_K_INTERSECTION:
+ q_wrapper->update_cb = &imap_intersection_ref;
+ break;
+ case CLERI_GID_C_DIFFERENCE:
+ q_wrapper->update_cb = &imap_difference_ref;
+ break;
+ case CLERI_GID_K_SYMMETRIC_DIFFERENCE:
+ q_wrapper->update_cb = &imap_symmetric_difference_ref;
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ SIRIPARSER_NEXT_NODE
+}
+
+typedef struct
+{
+ cexpr_t * where_expr;
+ siridb_tag_t * tag;
+} LISTENER_tag_t;
+
+static int LISTENER_tag__cb(siridb_series_t * series, LISTENER_tag_t * w)
+{
+ int rc = cexpr_run(
+ w->where_expr,
+ (cexpr_cb_t) siridb_series_cexpr_cb,
+ series);
+
+ if (rc == 0 || imap_add(w->tag->series, series->id, series) != 0)
+ {
+ siridb_series_decref(series);
+ }
+ return rc;
+}
+
+static int LISTENER_untag__cb(siridb_series_t * series, LISTENER_tag_t * w)
+{
+ int rc = cexpr_run(
+ w->where_expr,
+ (cexpr_cb_t) siridb_series_cexpr_cb,
+ series);
+
+ if (rc == 1 && imap_pop(w->tag->series, series->id) == series)
+ {
+ siridb_series_decref(series);
+ }
+
+ siridb_series_decref(series);
+ return rc;
+}
+
+static void enter_tag_series(uv_async_t * handle)
+{
+
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+ MASTER_CHECK_VERSION(siridb, "2.0.38")
+
+ cleri_node_t * tag_node =
+ query->nodes->node->children->next->node;
+ siridb_tag_t * tag;
+ char name[tag_node->len - 1];
+ xstr_extract_string(name, tag_node->str, tag_node->len);
+
+ tag = ct_get(siridb->tags->tags, name);
+ if (tag == NULL)
+ {
+ if (ct_get(siridb->groups->groups, name) != NULL)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot create tag `%s` because a group with this name "
+ "already exist.",
+ name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ if (siridb_tag_check_name(name, query->err_msg) != 0)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ uv_mutex_lock(&siridb->tags->mutex);
+
+ tag = siridb_tags_add(siridb->tags, name);
+
+ if (tag == NULL)
+ {
+ uv_mutex_unlock(&siridb->tags->mutex);
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Unexpected error while creating tag: `%s`",
+ name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ }
+ else
+ {
+ uv_mutex_lock(&siridb->tags->mutex);
+ }
+
+ if (q_alter->where_expr == NULL)
+ {
+ q_alter->n = q_alter->series_map->len;
+
+ imap_union_ref(
+ tag->series,
+ q_alter->series_map,
+ (imap_free_cb) &siridb__series_decref);
+ }
+ else
+ {
+ LISTENER_tag_t w = {
+ .where_expr = q_alter->where_expr,
+ .tag = tag,
+ };
+
+ q_alter->n = imap_walk(
+ q_alter->series_map, (imap_cb) LISTENER_tag__cb, &w);
+
+ imap_free(q_alter->series_map, NULL);
+ }
+
+ siridb_tags_set_require_save(siridb->tags, tag);
+
+ uv_mutex_unlock(&siridb->tags->mutex);
+
+ q_alter->series_map = NULL;
+
+ QP_ADD_SUCCESS
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_tag_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_alter->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void enter_timeit_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query->timeit = qp_packer_new(512);
+
+ if (query->timeit == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_raw(query->timeit, (const unsigned char *) "__timeit__", 10);
+ qp_add_type(query->timeit, QP_ARRAY_OPEN);
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void enter_untag_series(uv_async_t * handle)
+{
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+
+ q_alter->tp = QUERY_ALTER_SERIES;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+ MASTER_CHECK_VERSION(siridb, "2.0.38")
+
+ cleri_node_t * tag_node =
+ query->nodes->node->children->next->node;
+ siridb_tag_t * tag;
+
+ char name[tag_node->len - 1];
+ xstr_extract_string(name, tag_node->str, tag_node->len);
+
+ tag = ct_get(siridb->tags->tags, name);
+
+ if (tag == NULL)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find tag: '%s'",
+ name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ uv_mutex_lock(&siridb->tags->mutex);
+
+ if (q_alter->where_expr == NULL)
+ {
+ q_alter->n = q_alter->series_map->len;
+
+ imap_difference_ref(
+ tag->series,
+ q_alter->series_map,
+ (imap_free_cb) &siridb__series_decref);
+ }
+ else
+ {
+ LISTENER_tag_t w = {
+ .where_expr = q_alter->where_expr,
+ .tag = tag,
+ };
+
+ q_alter->n = imap_walk(
+ q_alter->series_map, (imap_cb) LISTENER_untag__cb, &w);
+
+ imap_free(q_alter->series_map, NULL);
+ }
+
+
+ siridb_tags_set_require_save(siridb->tags, tag);
+
+ uv_mutex_unlock(&siridb->tags->mutex);
+
+ q_alter->series_map = NULL;
+
+ QP_ADD_SUCCESS
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_tag_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_alter->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void enter_where_xxx(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ cexpr_t * cexpr =
+ cexpr_from_node(query->nodes->node->children->next->node);
+
+ if (cexpr == NULL)
+ {
+ sprintf(query->err_msg, "Max depth reached in 'where' expression!");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ ((query_wrapper_t *) query->data)->where_expr = cexpr;
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void enter_xxx_columns(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ cleri_children_t * columns = query->nodes->node->children;
+ query_list_t * qlist = (query_list_t *) query->data;
+
+ qlist->props = vec_new(DEFAULT_ALLOC_COLUMNS);
+
+ if (qlist->props == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ while (1)
+ {
+ qp_add_raw(
+ query->packer,
+ (const unsigned char *) columns->node->str,
+ columns->node->len);
+
+ if (vec_append_safe(
+ &qlist->props,
+ &columns->node->children->node->cl_obj->gid))
+ {
+ MEM_ERR_RET
+ }
+
+ if (columns->next == NULL)
+ {
+ break;
+ }
+
+ columns = columns->next->next;
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+/******************************************************************************
+ * Exit functions
+ *****************************************************************************/
+
+static void exit_after_expr(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ ((query_select_t *) query->data)->start_ts =
+ (uint64_t *) CLERI_NODE_DATA_ADDR(
+ query->nodes->node->children->next->node);
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void exit_alter_group(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ if (siridb_groups_save(siridb->groups))
+ {
+ FILE_ERR_RET
+ }
+
+ QP_ADD_SUCCESS
+ char * name = ((query_alter_t *) query->data)->via.group->name;
+ log_info(MSG_SUCCESS_ALTER_GROUP, name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_ALTER_GROUP, name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_alter_tag(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_tag_t * tag = ((query_alter_t *) query->data)->via.tag;
+
+ siridb_tags_set_require_save(siridb->tags, tag);
+
+ QP_ADD_SUCCESS
+
+ log_info(MSG_SUCCESS_ALTER_TAG, tag->name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_ALTER_TAG, tag->name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_alter_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ if (siridb_users_save(siridb))
+ {
+ FILE_ERR_RET
+ }
+
+ QP_ADD_SUCCESS
+ char * name = ((query_alter_t *) query->data)->via.user->name;
+ log_info(MSG_SUCCESS_ALTER_USER, name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_ALTER_USER, name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_before_expr(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ ((query_select_t *) query->data)->end_ts =
+ (uint64_t *) CLERI_NODE_DATA_ADDR(
+ query->nodes->node->children->next->node);
+
+ SIRIPARSER_NEXT_NODE
+}
+
+static void exit_between_expr(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_select_t * q_select = query->data;
+
+ q_select->start_ts = (uint64_t *) CLERI_NODE_DATA_ADDR(
+ query->nodes->node->children->next->node);
+
+ q_select->end_ts = (uint64_t *) CLERI_NODE_DATA_ADDR(
+ query->nodes->node->children->next->next->next->node);
+
+ if (*q_select->start_ts > *q_select->end_ts)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Start time (%" PRIu64 ") "
+ "should not be greater than end time (%" PRIu64 ")",
+ *q_select->start_ts,
+ *q_select->end_ts);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ SIRIPARSER_NEXT_NODE
+ }
+}
+
+static void exit_calc_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ cleri_node_t * calc_node = query->nodes->node;
+
+ assert (query->packer == NULL);
+
+ query->packer = sirinet_packer_new(64);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+ qp_add_raw(query->packer, (const unsigned char *) "calc", 4);
+
+ if (!query->factor)
+ {
+ qp_add_int64(query->packer, CLERI_NODE_DATA(calc_node));
+ }
+ else
+ {
+ double factor = (double) query->factor;
+ qp_add_int64(
+ query->packer,
+ (int64_t) ((CLERI_NODE_DATA(calc_node) * factor)));
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_count_groups(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+
+ if (q_count->where_expr == NULL || !cexpr_contains(
+ q_count->where_expr,
+ siridb_group_is_remote_prop))
+ {
+ finish_count_groups(handle);
+ }
+ else
+ {
+ sirinet_pkg_t * pkg = sirinet_pkg_new(0, 0, BPROTO_REQ_GROUPS, NULL);
+
+ if (pkg != NULL)
+ {
+ siri_async_incref(handle);
+
+ query->nodes->cb = (uv_async_cb) finish_count_groups;
+
+ siridb_pools_send_pkg(
+ siridb,
+ pkg,
+ 0,
+ (sirinet_promises_cb) on_groups_response,
+ handle,
+ 0);
+ }
+ }
+}
+
+static void exit_count_pools(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+ siridb_pool_t * pool = siridb->pools->pool + siridb->server->pool;
+
+ siridb_pool_walker_t wpool = {
+ .servers=pool->len,
+ .series=siridb->series->len,
+ .pool=siridb->server->pool
+ };
+
+ qp_add_raw(query->packer, (const unsigned char *) "pools", 5);
+
+ if (q_count->where_expr == NULL)
+ {
+ qp_add_int64(query->packer, siridb->pools->len);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ else
+ {
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ q_count->n = cexpr_run(
+ q_count->where_expr,
+ (cexpr_cb_t) siridb_pool_cexpr_cb,
+ &wpool);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_POOLS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+
+static void exit_count_series(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+
+ MASTER_CHECK_ONLINE(siridb)
+
+ qp_add_raw(query->packer, (const unsigned char *) "series", 6);
+
+ if (q_count->where_expr == NULL)
+ {
+ q_count->n = (q_count->series_map == NULL) ?
+ siridb->series_map->len : q_count->series_map->len;
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_POOLS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+ else
+ {
+ uv_mutex_lock(&siridb->series_mutex);
+
+ q_count->vec = imap_2vec_ref(
+ (q_count->series_map == NULL) ?
+ siridb->series_map : q_count->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (q_count->vec == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ uv_async_t * next = malloc(sizeof(uv_async_t));
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) async_count_series);
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+ }
+}
+
+static void exit_count_series_length(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ qp_add_raw(query->packer, (const unsigned char *) "series_length", 13);
+
+ if (q_count->where_expr == NULL)
+ {
+ size_t i;
+ vec_t * vec;
+ siridb_series_t * series;
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ vec = imap_2vec((q_count->series_map == NULL) ?
+ siridb->series_map : q_count->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (vec == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ for (i = 0; i < vec->len; i++)
+ {
+ series = (siridb_series_t *) vec->data[i];
+ q_count->n += series->length;
+ }
+
+ vec_free(vec);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_POOLS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+ else
+ {
+ uv_async_t * next;
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ q_count->vec = imap_2vec_ref(
+ (q_count->series_map == NULL) ?
+ siridb->series_map : q_count->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (q_count->vec == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next = malloc(sizeof(uv_async_t));
+
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) async_count_series_length);
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+ }
+}
+
+static void exit_count_servers(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+ cexpr_t * where_expr = q_count->where_expr;
+ cexpr_cb_t cb = (cexpr_cb_t) siridb_server_cexpr_cb;
+ int is_local = IS_MASTER;
+
+ qp_add_raw(query->packer, (const unsigned char *) "servers", 7);
+
+
+ /* if is_local, check if we use 'remote' props in where expression */
+ if (is_local && where_expr != NULL)
+ {
+ is_local = !cexpr_contains(where_expr, siridb_server_is_remote_prop);
+ }
+
+ if (is_local)
+ {
+ llist_node_t * node;
+ for ( node = siridb->servers->first;
+ node != NULL;
+ node = node->next)
+ {
+ siridb_server_walker_t wserver = {node->data, siridb};
+ if (where_expr == NULL || cexpr_run(where_expr, cb, &wserver))
+ {
+ q_count->n++;
+ }
+ }
+ }
+ else
+ {
+ siridb_server_walker_t wserver = {siridb->server, siridb};
+ if (where_expr == NULL || cexpr_run(where_expr, cb, &wserver))
+ {
+ q_count->n++;
+ }
+ }
+
+ if (IS_MASTER && !is_local)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_count_servers_received(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+ cexpr_t * where_expr = q_count->where_expr;
+ cexpr_cb_t cb = (cexpr_cb_t) siridb_server_cexpr_cb;
+
+ qp_add_raw(
+ query->packer,
+ (const unsigned char *) "servers_received_points",
+ 23);
+
+ siridb_server_walker_t wserver = {siridb->server, siridb};
+ if (where_expr == NULL || cexpr_run(where_expr, cb, &wserver))
+ {
+ q_count->n += siridb->received_points;
+ }
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_count_servers_selected(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+ cexpr_t * where_expr = q_count->where_expr;
+ cexpr_cb_t cb = (cexpr_cb_t) siridb_server_cexpr_cb;
+
+ qp_add_raw(
+ query->packer,
+ (const unsigned char *) "servers_selected_points",
+ 23);
+
+ siridb_server_walker_t wserver = {siridb->server, siridb};
+ if (where_expr == NULL || cexpr_run(where_expr, cb, &wserver))
+ {
+ q_count->n += siridb->selected_points;
+ }
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_count_shards(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+
+ qp_add_raw(query->packer, (const unsigned char *) "shards", 6);
+
+ if (q_count->where_expr == NULL)
+ {
+ q_count->n = siridb_shards_n(siridb);
+ }
+ else
+ {
+ uint64_t duration;
+ siridb_shard_view_t vshard = {
+ .server=siridb->server
+ };
+ size_t i;
+ vec_t * shards_list;
+
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ shards_list = siridb_shards_vec(siridb);
+
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ if (shards_list == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ for (i = 0; i < shards_list->len; i++)
+ {
+ vshard.shard = (siridb_shard_t *) shards_list->data[i];
+
+ /* set start and end properties */
+ duration = vshard.shard->duration;
+ vshard.start = vshard.shard->id - vshard.shard->id % duration;
+ vshard.end = vshard.start + duration;
+
+ if (cexpr_run(
+ q_count->where_expr,
+ (cexpr_cb_t) siridb_shard_cexpr_cb,
+ &vshard))
+ {
+ q_count->n++;
+ }
+
+ siridb_shard_decref(vshard.shard);
+ }
+
+ vec_free(shards_list);
+ }
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_count_shards_size(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+ uint64_t duration;
+ size_t i;
+ vec_t * shards_list;
+ siridb_shard_view_t vshard = {
+ .server=siridb->server
+ };
+
+ qp_add_raw(query->packer, (const unsigned char *) "shards_size", 11);
+
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ shards_list = siridb_shards_vec(siridb);
+
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ if (shards_list == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ for (i = 0; i < shards_list->len; i++)
+ {
+ vshard.shard = (siridb_shard_t *) shards_list->data[i];
+
+ /* set start and end properties */
+ duration = vshard.shard->duration;
+
+ vshard.start = vshard.shard->id - vshard.shard->id % duration;
+ vshard.end = vshard.start + duration;
+
+ if (q_count->where_expr == NULL || cexpr_run(
+ q_count->where_expr,
+ (cexpr_cb_t) siridb_shard_cexpr_cb,
+ &vshard))
+ {
+ q_count->n += vshard.shard->len;
+ }
+
+ siridb_shard_decref(vshard.shard);
+ }
+
+ vec_free(shards_list);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_count_tags(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_count_t * q_count = (query_count_t *) query->data;
+
+ if (q_count->where_expr == NULL || !cexpr_contains(
+ q_count->where_expr,
+ siridb_tag_is_remote_prop))
+ {
+ finish_count_tags(handle);
+ }
+ else
+ {
+ sirinet_pkg_t * pkg = sirinet_pkg_new(0, 0, BPROTO_REQ_TAGS, NULL);
+
+ if (pkg != NULL)
+ {
+ siri_async_incref(handle);
+
+ query->nodes->cb = (uv_async_cb) finish_count_tags;
+
+ siridb_pools_send_pkg(
+ siridb,
+ pkg,
+ 0,
+ (sirinet_promises_cb) on_tags_response,
+ handle,
+ 0);
+ }
+ }
+}
+
+static void exit_count_users(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ llist_node_t * node = siridb->users->first;
+ cexpr_t * where_expr = ((query_count_t *) query->data)->where_expr;
+ cexpr_cb_t cb = (cexpr_cb_t) siridb_user_cexpr_cb;
+ int n = 0;
+
+ qp_add_raw(query->packer, (const unsigned char *) "users", 5);
+
+ while (node != NULL)
+ {
+ if (where_expr == NULL || cexpr_run(where_expr, cb, node->data))
+ {
+ n++;
+ }
+ node = node->next;
+ }
+
+ qp_add_int64(query->packer, n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_create_group(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ cleri_node_t * name_nd =
+ query->nodes->node->children->next->node;
+
+ cleri_node_t * for_nd =
+ query->nodes->node->children->next->next->next->node;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ char group_name[name_nd->len - 1];
+ xstr_extract_string(group_name, name_nd->str, name_nd->len);
+
+ if (siridb_groups_add_group(
+ siridb,
+ group_name,
+ for_nd->str,
+ for_nd->len,
+ query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else if (siridb_groups_save(siridb->groups))
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot write groups to file: %s",
+ siridb->groups->fn);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ log_critical("%s", query->err_msg);
+ }
+ else
+ {
+ assert (query->packer == NULL);
+ query->packer = sirinet_packer_new(1024);
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ QP_ADD_SUCCESS
+ log_info(MSG_SUCCESS_CREATE_GROUP, group_name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_CREATE_GROUP, group_name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+
+static void exit_create_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_user_t * user = ((query_alter_t *) query->data)->via.user;
+ cleri_node_t * user_node =
+ query->nodes->node->children->next->node;
+
+ /* both name and packer should be NULL at this point */
+ assert(user->name == NULL);
+ assert(query->packer == NULL);
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ char name[user_node->len - 1];
+ xstr_extract_string(name, user_node->str, user_node->len);
+
+ if (siridb_user_set_name(
+ siridb,
+ user,
+ name,
+ query->err_msg) ||
+ siridb_users_add_user(
+ siridb,
+ user,
+ query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ /* success, increment the user reference counter */
+ siridb_user_incref(user);
+
+ assert (query->packer == NULL);
+ query->packer = sirinet_packer_new(1024);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ QP_ADD_SUCCESS
+
+ log_info(MSG_SUCCESS_CREATE_USER, user->name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_CREATE_USER, user->name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+
+static void exit_drop_group(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * group_node =
+ query->nodes->node->children->next->node;
+
+ char name[group_node->len - 1];
+
+ xstr_extract_string(name, group_node->str, group_node->len);
+
+ if (siridb_groups_drop_group(siridb->groups, name, query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+ log_info(MSG_SUCCESS_DROP_GROUP, name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_DROP_GROUP, name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+
+static void exit_drop_series(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_drop_t * q_drop = (query_drop_t *) query->data;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ /*
+ * We transform or copy the references from imap to vec because we need
+ * this list for both filtering or performing the actual drop.
+ */
+ uv_mutex_lock(&siridb->series_mutex);
+
+ q_drop->vec = (q_drop->series_map == NULL) ?
+ imap_2vec_ref(siridb->series_map) :
+ imap_vec_pop(q_drop->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (q_drop->vec == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ if (q_drop->series_map != NULL)
+ {
+ /* now we can simply destroy the imap in case we had one */
+ imap_free(q_drop->series_map, NULL);
+ q_drop->series_map = NULL;
+ }
+
+ /*
+ * This function will be called twice when using a where statement.
+ * The second time the where_expr is NULL and the reason we do this is
+ * so that we can honor a correct drop threshold.
+ */
+ if (q_drop->where_expr != NULL)
+ {
+ /* create a new one */
+ q_drop->series_map = imap_new();
+
+ if (q_drop->series_map == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ uv_async_t * next = malloc(sizeof(uv_async_t));
+
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) async_filter_series);
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+
+ }
+ else
+ {
+ double percent = siridb->series_map->len
+ ? (double) q_drop->vec->len / siridb->series_map->len
+ : 0.0;
+
+ if (IS_MASTER &&
+ q_drop->vec->len &&
+ (~q_drop->flags & QUERIES_IGNORE_DROP_THRESHOLD) &&
+ percent >= siridb->drop_threshold)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "This query would drop %0.2f%% of the series in pool %u. "
+ "Add \'set ignore_threshold true\' to the query "
+ "statement if you really want to do this.",
+ percent * 100,
+ siridb->server->pool);
+
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+
+ q_drop->n = q_drop->vec->len;
+
+ uv_async_t * next = malloc(sizeof(uv_async_t));
+
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) async_drop_series);
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+ }
+ }
+}
+
+static void exit_drop_server(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_server_t * server = siridb_server_from_node(
+ siridb,
+ query->nodes->node->children->next->node->children->node,
+ query->err_msg);
+
+ MASTER_CHECK_REINDEXING(siridb)
+ MASTER_CHECK_ONLINE(siridb)
+
+ if (server == NULL)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else if (siridb->pools->pool[server->pool].len == 1)
+ {
+ snprintf(
+ query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot remove server '%s' because this is the only "
+ "server for pool %u",
+ server->name,
+ server->pool);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else if (siridb->server == server || server->client != NULL)
+ {
+ snprintf(
+ query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot remove server '%s' because the server is still "
+ "online. (stop SiriDB on this server if you really want "
+ "to remove the server)",
+ server->name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+ log_info(MSG_SUCCESS_DROP_SERVER, server->name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_DROP_SERVER, server->name);
+
+ if (IS_MASTER)
+ {
+ query->flags |= SIRIDB_QUERY_FLAG_REBUILD;
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ FLAG_ONLY_CHECK_ONLINE);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+
+ /*
+ * After calling this function, all references to the server
+ * object might be gone
+ */
+ if (siridb_server_drop(siridb, server))
+ {
+ log_critical("Cannot save servers to file");
+ }
+ }
+}
+
+static void exit_drop_shards(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_drop_t * q_drop = (query_drop_t *) query->data;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ q_drop->shards_list = siridb_shards_vec(siridb);
+
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ if (q_drop->shards_list == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ if (q_drop->where_expr != NULL)
+ {
+ uint64_t duration;
+ siridb_shard_view_t vshard = {
+ .server=siridb->server
+ };
+ size_t dropped = 0;
+ size_t i;
+
+ for (i = 0; i < q_drop->shards_list->len; i++)
+ {
+ vshard.shard = (siridb_shard_t *) q_drop->shards_list->data[i];
+
+ /* set start and end properties */
+ duration = vshard.shard->duration;
+
+ vshard.start = vshard.shard->id - vshard.shard->id % duration;
+ vshard.end = vshard.start + duration;
+
+ if (!cexpr_run(
+ q_drop->where_expr,
+ (cexpr_cb_t) siridb_shard_cexpr_cb,
+ &vshard))
+ {
+ siridb_shard_decref(vshard.shard);
+ dropped++;
+ }
+ else if (dropped)
+ {
+ q_drop->shards_list->data[i - dropped] = vshard.shard;
+ }
+ }
+
+ q_drop->shards_list->len -= dropped;
+ }
+
+ size_t n = siridb_shards_n(siridb);
+ double percent = n
+ ? (double) q_drop->shards_list->len / n
+ : 0.0;
+
+ if (IS_MASTER &&
+ q_drop->shards_list->len &&
+ (~q_drop->flags & QUERIES_IGNORE_DROP_THRESHOLD) &&
+ percent >= siridb->drop_threshold)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "This query would drop %0.2f%% of the shards in pool %u. "
+ "Add \'set ignore_threshold true\' to the query "
+ "statement if you really want to do this.",
+ percent * 100,
+ siridb->server->pool);
+
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ uv_async_t * next;
+
+ QP_ADD_SUCCESS
+
+ q_drop->n = q_drop->shards_list->len;
+
+ next = malloc(sizeof(uv_async_t));
+
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) async_drop_shards);
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+ }
+}
+
+static void exit_drop_tag(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * tag_node =
+ query->nodes->node->children->next->node;
+
+ char name[tag_node->len - 1];
+
+ xstr_extract_string(name, tag_node->str, tag_node->len);
+
+ if (siridb_tags_drop_tag(siridb->tags, name, query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+ log_info(MSG_SUCCESS_DROP_TAG, name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_DROP_TAG, name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+
+static void exit_drop_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * user_node =
+ query->nodes->node->children->next->node;
+ char username[user_node->len - 1];
+
+ xstr_extract_string(username, user_node->str, user_node->len);
+
+ if (siridb_users_drop_user(
+ siridb,
+ username,
+ query->err_msg))
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+ log_info(MSG_SUCCESS_DROP_USER, username);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_DROP_USER, username);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+
+static void exit_grant_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ if (siridb_users_save(siridb))
+ {
+ sprintf(query->err_msg, "Could not write users to file!");
+ log_critical(query->err_msg);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ assert (query->packer == NULL);
+
+ query->packer = sirinet_packer_new(1024);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ QP_ADD_SUCCESS
+ char * name = ((query_alter_t *) query->data)->via.user->name;
+ log_info(MSG_SUCCESS_GRANT_USER, name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_GRANT_USER, name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_help_xxx(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ if (query->data != NULL)
+ {
+ assert (query->packer == NULL);
+ const char * help = siri_help_get(
+ query->nodes->node->cl_obj->gid,
+ (const char *) query->data,
+ query->err_msg);
+
+ if (help == NULL)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ query->packer = sirinet_packer_new(4096);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+ qp_add_raw(query->packer, (const unsigned char *) "help", 4);
+ qp_add_string(query->packer, help);
+
+ free(query->data);
+ query->data = NULL;
+ }
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_list_groups(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_list_t * q_list = (query_list_t *) query->data;
+
+ int is_local = (q_list->props == NULL);
+
+ /* if not is_local check for 'remote' columns */
+ if (!is_local)
+ {
+ size_t i;
+ is_local = 1;
+ for (i = 0; i < q_list->props->len; i++)
+ {
+ if (siridb_group_is_remote_prop(
+ *((uint32_t *) q_list->props->data[i])))
+ {
+ is_local = 0;
+ break;
+ }
+ }
+ }
+
+ /* if is_local, check if we use 'remote' props in where expression */
+ if (is_local && q_list->where_expr != NULL)
+ {
+ is_local = !cexpr_contains(
+ q_list->where_expr,
+ siridb_group_is_remote_prop);
+ }
+
+ if (is_local)
+ {
+ finish_list_groups(handle);
+ }
+ else
+ {
+ sirinet_pkg_t * pkg = sirinet_pkg_new(0, 0, BPROTO_REQ_GROUPS, NULL);
+
+ if (pkg != NULL)
+ {
+ siri_async_incref(handle);
+
+ query->nodes->cb = (uv_async_cb) finish_list_groups;
+
+ siridb_pools_send_pkg(
+ siridb,
+ pkg,
+ 0,
+ (sirinet_promises_cb) on_groups_response,
+ handle,
+ 0);
+ }
+ }
+}
+
+static void exit_list_pools(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_list_t * q_list = (query_list_t *) query->data;
+ siridb_pool_t * pool = siridb->pools->pool + siridb->server->pool;
+ siridb_pool_walker_t wpool = {
+ .servers=pool->len,
+ .series=siridb->series->len,
+ .pool=siridb->server->pool
+ };
+ uint_fast16_t prop;
+ cexpr_t * where_expr = q_list->where_expr;
+
+ if (q_list->props == NULL)
+ {
+ q_list->props = vec_new(3);
+
+ if (q_list->props == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ vec_append(q_list->props, &GID_K_POOL);
+ vec_append(q_list->props, &GID_K_SERVERS);
+ vec_append(q_list->props, &GID_K_SERIES);
+ qp_add_raw(query->packer, (const unsigned char *) "pool", 4);
+ qp_add_raw(query->packer, (const unsigned char *) "servers", 7);
+ qp_add_raw(query->packer, (const unsigned char *) "series", 6);
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ qp_add_raw(query->packer, (const unsigned char *) "pools", 5);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ if ( q_list->limit &&
+ (where_expr == NULL || cexpr_run(
+ where_expr,
+ (cexpr_cb_t) siridb_pool_cexpr_cb,
+ &wpool)))
+ {
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ for (prop = 0; prop < q_list->props->len; prop++)
+ {
+ switch(*((uint32_t *) q_list->props->data[prop]))
+ {
+ case CLERI_GID_K_POOL:
+ qp_add_int64(query->packer, (int64_t) wpool.pool);
+ break;
+ case CLERI_GID_K_SERVERS:
+ qp_add_int64(query->packer, (int64_t) wpool.servers);
+ break;
+ case CLERI_GID_K_SERIES:
+ qp_add_int64(query->packer, (int64_t) wpool.series);
+ break;
+ }
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+ q_list->limit--;
+ }
+
+ if (IS_MASTER && q_list->limit)
+ {
+ /* we have not reached the limit, send the query to other pools */
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_POOLS,
+ (sirinet_promises_cb) on_list_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_list_series(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_list_t * q_list = (query_list_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ if (q_list->props == NULL)
+ {
+ q_list->props = vec_new(1);
+
+ if (q_list->props == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ vec_append(q_list->props, &GID_K_NAME);
+ qp_add_raw(query->packer, (const unsigned char *) "name", 4);
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ qp_add_raw(query->packer, (const unsigned char *) "series", 6);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ q_list->vec = imap_2vec_ref((q_list->series_map == NULL) ?
+ siridb->series_map : q_list->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (q_list->vec == NULL)
+ {
+ MEM_ERR_RET;
+ }
+
+ uv_async_t * next = malloc(sizeof(uv_async_t));
+
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) async_list_series);
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+}
+
+static void exit_list_servers(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ query_list_t * q_list = (query_list_t *) query->data;
+ cexpr_t * where_expr = q_list->where_expr;
+ int is_local = IS_MASTER;
+
+ /* if is_local, check if we need ask for 'remote' columns */
+ if (is_local && q_list->props != NULL)
+ {
+ size_t i;
+ for (i = 0; i < q_list->props->len; i++)
+ {
+ if (siridb_server_is_remote_prop(
+ *((uint32_t *) q_list->props->data[i])))
+ {
+ is_local = 0;
+ break;
+ }
+ }
+ }
+
+ /* if is_local, check if we use 'remote' props in where expression */
+ if (is_local && where_expr != NULL)
+ {
+ is_local = !cexpr_contains(where_expr, siridb_server_is_remote_prop);
+ }
+
+ if (q_list->props == NULL)
+ {
+ q_list->props = vec_new(5);
+
+ if (q_list->props == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ vec_append(q_list->props, &GID_K_NAME);
+ vec_append(q_list->props, &GID_K_POOL);
+ vec_append(q_list->props, &GID_K_VERSION);
+ vec_append(q_list->props, &GID_K_ONLINE);
+ vec_append(q_list->props, &GID_K_STATUS);
+ qp_add_raw(query->packer, (const unsigned char *) "name", 4);
+ qp_add_raw(query->packer, (const unsigned char *) "pool", 4);
+ qp_add_raw(query->packer, (const unsigned char *) "version", 7);
+ qp_add_raw(query->packer, (const unsigned char *) "online", 6);
+ qp_add_raw(query->packer, (const unsigned char *) "status", 6);
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ qp_add_raw(query->packer, (const unsigned char *) "servers", 7);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ if (is_local)
+ {
+ llist_walkn(
+ siridb->servers,
+ &q_list->limit,
+ (llist_cb) siridb_servers_list,
+ handle);
+ }
+ else
+ {
+ q_list->limit -= siridb_servers_list(siridb->server, handle);
+ }
+
+
+ if (IS_MASTER && !is_local && q_list->limit)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_list_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_list_shards(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ query_list_t * q_list = (query_list_t *) query->data;
+ uint_fast16_t prop;
+ uint64_t duration;
+ cexpr_t * where_expr = q_list->where_expr;
+ siridb_shard_view_t vshard = {
+ .server=siridb->server
+ };
+ size_t i;
+ vec_t * shards_list;
+
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ shards_list = siridb_shards_vec(siridb);
+
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ if (shards_list == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ if (q_list->props == NULL)
+ {
+ q_list->props = vec_new(5);
+
+ if (q_list->props == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ vec_append(q_list->props, &GID_K_SID);
+ vec_append(q_list->props, &GID_K_POOL);
+ vec_append(q_list->props, &GID_K_SERVER);
+ vec_append(q_list->props, &GID_K_START);
+ vec_append(q_list->props, &GID_K_END);
+ qp_add_raw(query->packer, (const unsigned char *) "sid", 3);
+ qp_add_raw(query->packer, (const unsigned char *) "pool", 4);
+ qp_add_raw(query->packer, (const unsigned char *) "server", 6);
+ qp_add_raw(query->packer, (const unsigned char *) "start", 5);
+ qp_add_raw(query->packer, (const unsigned char *) "end", 3);
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ qp_add_raw(query->packer, (const unsigned char *) "shards", 6);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ for (i = 0; i < shards_list->len; i++)
+ {
+ vshard.shard = (siridb_shard_t *) shards_list->data[i];
+
+ /* set start and end properties */
+ duration = vshard.shard->duration;
+
+ vshard.start = vshard.shard->id - vshard.shard->id % duration;
+ vshard.end = vshard.start + duration;
+
+ if (q_list->limit && (where_expr == NULL || cexpr_run(
+ where_expr,
+ (cexpr_cb_t) siridb_shard_cexpr_cb,
+ &vshard)))
+ {
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ for (prop = 0; prop < q_list->props->len; prop++)
+ {
+ switch(*((uint32_t *) q_list->props->data[prop]))
+ {
+ case CLERI_GID_K_SID:
+ qp_add_int64(query->packer, (int64_t) vshard.shard->id);
+ break;
+ case CLERI_GID_K_POOL:
+ qp_add_int64(query->packer, (int64_t) vshard.server->pool);
+ break;
+ case CLERI_GID_K_SIZE:
+ qp_add_int64(query->packer, (int64_t) vshard.shard->len);
+ break;
+ case CLERI_GID_K_START:
+ qp_add_int64(query->packer, (int64_t) vshard.start);
+ break;
+ case CLERI_GID_K_END:
+ qp_add_int64(query->packer, (int64_t) vshard.end);
+ break;
+ case CLERI_GID_K_TYPE:
+ qp_add_string(
+ query->packer,
+ shard_type_map[vshard.shard->tp]);
+ break;
+ case CLERI_GID_K_SERVER:
+ qp_add_string(query->packer, vshard.server->name);
+ break;
+ case CLERI_GID_K_STATUS:
+ {
+ char buffer[SIRIDB_SHARD_STATUS_STR_MAX];
+ int n = siridb_shard_status(buffer, vshard.shard);
+ qp_add_raw(
+ query->packer,
+ (const unsigned char *) buffer,
+ n);
+ }
+ break;
+ }
+ }
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ q_list->limit--;
+ }
+
+ siridb_shard_decref(vshard.shard);
+ }
+
+ vec_free(shards_list);
+
+ if (IS_MASTER && q_list->limit)
+ {
+ /* we have not reached the limit, send the query to other pools */
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_list_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_list_tags(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ query_list_t * q_list = (query_list_t *) query->data;
+
+ int is_local = (q_list->props == NULL);
+
+ /* if not is_local check for 'remote' columns */
+ if (!is_local)
+ {
+ size_t i;
+ is_local = 1;
+ for (i = 0; i < q_list->props->len; i++)
+ {
+ if (siridb_tag_is_remote_prop(
+ *((uint32_t *) q_list->props->data[i])))
+ {
+ is_local = 0;
+ break;
+ }
+ }
+ }
+
+ /* if is_local, check if we use 'remote' props in where expression */
+ if (is_local && q_list->where_expr != NULL)
+ {
+ is_local = !cexpr_contains(
+ q_list->where_expr,
+ siridb_tag_is_remote_prop);
+ }
+
+ if (is_local)
+ {
+ finish_list_tags(handle);
+ }
+ else
+ {
+ sirinet_pkg_t * pkg = sirinet_pkg_new(0, 0, BPROTO_REQ_TAGS, NULL);
+
+ if (pkg != NULL)
+ {
+ siri_async_incref(handle);
+
+ query->nodes->cb = (uv_async_cb) finish_list_tags;
+
+ siridb_pools_send_pkg(
+ siridb,
+ pkg,
+ 0,
+ (sirinet_promises_cb) on_tags_response,
+ handle,
+ 0);
+ }
+ }
+}
+
+static void exit_list_users(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ llist_node_t * node = siridb->users->first;
+ vec_t * props = ((query_list_t *) query->data)->props;
+ cexpr_cb_t cb = (cexpr_cb_t) siridb_user_cexpr_cb;
+ query_list_t * q_list = (query_list_t *) query->data;
+ cexpr_t * where_expr = q_list->where_expr;
+ size_t i;
+ siridb_user_t * user;
+
+ if (props == NULL)
+ {
+ qp_add_raw(query->packer, (const unsigned char *) "name", 4);
+ qp_add_raw(query->packer, (const unsigned char *) "access", 6);
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ qp_add_raw(query->packer, (const unsigned char *) "users", 5);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ while (node != NULL && q_list->limit)
+ {
+ user = node->data;
+
+ if (where_expr == NULL || cexpr_run(where_expr, cb, user))
+ {
+ q_list->limit--;
+
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ if (props == NULL)
+ {
+ siridb_user_prop(user, query->packer, CLERI_GID_K_NAME);
+ siridb_user_prop(user, query->packer, CLERI_GID_K_ACCESS);
+ }
+ else
+ {
+ for (i = 0; i < props->len; i++)
+ {
+ siridb_user_prop(
+ user,
+ query->packer,
+ *((uint32_t *) props->data[i]));
+ }
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ }
+ node = node->next;
+ }
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_revoke_user(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ if (siridb_users_save(siridb))
+ {
+ sprintf(query->err_msg, "Could not write users to file!");
+ log_critical(query->err_msg);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ assert (query->packer == NULL);
+
+ query->packer = sirinet_packer_new(1024);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+
+ QP_ADD_SUCCESS
+ char * name = ((query_alter_t *) query->data)->via.user->name;
+ log_info(MSG_SUCCESS_REVOKE_USER, name);
+ qp_add_fmt_safe(query->packer, MSG_SUCCESS_REVOKE_USER, name);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void exit_series_match(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_wrapper_t * q_wrapper = query->data;
+
+ if (q_wrapper->tp == QUERIES_SELECT)
+ {
+ query_select_t * q_select = (query_select_t *) q_wrapper;
+ if ((q_select->flags & QUERIES_SKIP_GET_POINTS) &&
+ (q_select->start_ts != NULL || q_select->end_ts != NULL))
+ {
+ q_select->flags &= ~QUERIES_SKIP_GET_POINTS;
+ }
+
+ if ((~q_select->flags & QUERIES_SKIP_GET_POINTS) &&
+ q_select->nselects > 1)
+ {
+ /* We have more than one select request, let's use points caching.
+ * (Not critical, everything works if points_map is NULL) */
+ q_select->points_map = imap_new();
+ }
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_series_parentheses(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_wrapper_t * q_wrapper = query->data;
+
+ assert (q_wrapper->sset_vec);
+ assert (q_wrapper->sset_vec->len);
+
+ siridb_sset_t * sset = vec_pop(q_wrapper->sset_vec);
+
+ if (sset->update_cb != NULL)
+ {
+ assert (sset->series_map != NULL);
+ (*sset->update_cb)(
+ sset->series_map,
+ q_wrapper->series_map,
+ (imap_free_cb) &siridb__series_decref);
+ q_wrapper->series_map = sset->series_map;
+ }
+ else
+ {
+ assert (sset->series_map == NULL);
+ }
+
+ /* simple free to prevent cleaning sset->series_map */
+ free (sset);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_select_aggregate(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_select_t * q_select = query->data;
+
+ if (q_select->where_expr != NULL)
+ {
+ /* we transform the references from imap to vec */
+ q_select->vec = imap_vec_pop(q_select->series_map);
+
+ if (q_select->vec == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ /* now we can simply destroy the imap */
+ imap_free(q_select->series_map, NULL);
+
+ /* create a new one */
+ q_select->series_map = imap_new();
+
+ if (q_select->series_map == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ uv_async_t * next = malloc(sizeof(uv_async_t));
+
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) async_filter_series);
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+
+ }
+ else if (siridb_presuf_add(&q_select->presuf, query->nodes->node) == NULL)
+ {
+ MEM_ERR_RET
+ }
+ else
+ {
+ q_select->nselects--;
+
+ if (!siridb_presuf_is_unique(q_select->presuf))
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "When using multiple select methods, add a prefix "
+ "and/or suffix to the selection to make them unique.");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ if (q_select->merge_as != NULL)
+ {
+ vec_t * plist = vec_new(VEC_DEFAULT_SIZE);
+
+ if (plist == NULL || ct_add(
+ q_select->result,
+ siridb_presuf_name(
+ q_select->presuf,
+ q_select->merge_as,
+ strlen(q_select->merge_as)),
+ plist))
+ {
+ sprintf(query->err_msg,
+ "Error while merging points. Make sure the "
+ "destination series name is valid.");
+ vec_free(plist);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ }
+
+ if (q_select->series_map->len)
+ {
+ q_select->alist = siridb_aggregate_list(
+ query->nodes->node->children->node->children,
+ query->err_msg);
+ if (q_select->alist == NULL)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ q_select->vec = imap_2vec_ref(q_select->series_map);
+
+ if (q_select->vec == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ uv_async_t * next = malloc(sizeof(uv_async_t));
+
+ if (next == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next->data = handle->data;
+
+ uv_async_init(
+ siri.loop,
+ next,
+ (uv_async_cb) (
+ (q_select->flags & QUERIES_SKIP_GET_POINTS) ?
+ async_no_points_aggregate :
+ async_select_aggregate));
+ uv_async_send(next);
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+ }
+}
+
+static void exit_select_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_select_t * q_select = query->data;
+
+ if (IS_MASTER)
+ {
+ if (q_select->pmap == NULL || q_select->pmap->len)
+ {
+ /* we have not reached the limit, send the query to other pools */
+ siridb_query_forward(
+ handle,
+ (q_select->pmap == NULL) ?
+ SIRIDB_QUERY_FWD_POOLS :
+ SIRIDB_QUERY_FWD_SOME_POOLS,
+ (sirinet_promises_cb) on_select_response,
+ 0);
+ }
+ else
+ {
+ uv_work_t * work = malloc(sizeof(uv_work_t));
+ if (work == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ uv_async_t * next = malloc(sizeof(uv_async_t));
+ if (next == NULL)
+ {
+ free(work);
+ MEM_ERR_RET
+ }
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+
+ handle = next;
+ handle->data = query;
+ siridb_nodes_next(&query->nodes);
+
+ uv_async_init(
+ siri.loop,
+ handle,
+ (query->nodes == NULL) ?
+ (uv_async_cb) siridb_send_query_result :
+ (uv_async_cb) query->nodes->cb);
+
+ siri_async_incref(handle);
+ work->data = handle;
+ uv_queue_work(
+ siri.loop,
+ work,
+ &master_select_work,
+ &master_select_work_finish);
+ }
+ }
+ else
+ {
+ if (qp_add_raw(query->packer, (const unsigned char *) "select", 6) ||
+ qp_add_type(query->packer, QP_MAP_OPEN) ||
+ ct_items(
+ q_select->result,
+ (q_select->merge_as == NULL) ?
+ (ct_item_cb) &items_select_other
+ :
+ (ct_item_cb) &items_select_other_merge,
+ handle) ||
+ qp_add_type(query->packer, QP_MAP_CLOSE))
+ {
+ MEM_ERR_RET
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+
+static void exit_set_address(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_server_t * server = ((query_alter_t *) query->data)->via.server;
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+ siridb_t * siridb = query->client->siridb;
+
+ if (siridb->server == server || server->client != NULL)
+ {
+ sprintf(query->err_msg, MSG_ERR_SERVER_ADDRESS);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ char address[node->len - 1];
+ xstr_extract_string(address, node->str, node->len);
+
+ int rc;
+ rc = siridb_server_update_address(siridb, server, address, server->port);
+ switch (rc)
+ {
+ case -1:
+ sprintf(query->err_msg, "Error while updating server address");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ break;
+
+ case 0:
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Server address is already set to '%s'",
+ server->address);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ break;
+
+ case 1:
+ QP_ADD_SUCCESS
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCESS_SET_ADDR_PORT,
+ server->name);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ break;
+ }
+}
+
+static void exit_set_backup_mode(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ assert (query->data != NULL);
+ assert (IS_MASTER);
+
+ siridb_server_t * server = ((query_alter_t *) query->data)->via.server;
+
+ int backup_mode = query->nodes->node->children->next->next->node->
+ children->node->cl_obj->gid == CLERI_GID_K_TRUE;
+
+ if (backup_mode ^ ((server->flags & SERVER_FLAG_BACKUP_MODE) != 0))
+ {
+ QP_ADD_SUCCESS
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCES_SET_BACKUP_MODE,
+ (backup_mode) ? "enabled" : "disabled",
+ server->name);
+
+ if (server == siridb->server)
+ {
+ if (backup_mode)
+ {
+ if (siri_backup_enable(&siri, siridb))
+ {
+ MEM_ERR_RET
+ }
+ }
+ else
+ {
+ if (siri_backup_disable(&siri, siridb))
+ {
+ MEM_ERR_RET
+ }
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ else
+ {
+ if (siridb_server_is_online(server))
+ {
+ sirinet_pkg_t * pkg = sirinet_pkg_new(
+ 0,
+ 0,
+ (backup_mode) ?
+ BPROTO_ENABLE_BACKUP_MODE :
+ BPROTO_DISABLE_BACKUP_MODE,
+ NULL);
+
+ if (pkg == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ /* handle will be bound to a timer so we should increment */
+ siri_async_incref(handle);
+
+ if (siridb_server_send_pkg(
+ server,
+ pkg,
+ 0,
+ (sirinet_promise_cb) on_ack_response,
+ handle,
+ 0))
+ {
+ /*
+ * signal is raised and 'on_ack_response' will not be
+ * called
+ */
+ free(pkg);
+ siri_async_decref(&handle);
+ MEM_ERR_RET
+ }
+ }
+ else
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot %s backup mode, '%s' is currently unavailable",
+ (backup_mode) ? "enable" : "disable",
+ server->name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ }
+ }
+ else
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Backup mode is already %s on '%s'.",
+ (backup_mode) ? "enabled" : "disabled",
+ server->name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+}
+
+static void exit_set_drop_threshold(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+
+ double drop_threshold = xstr_to_double(node->str);
+
+ if (drop_threshold < 0.0 || drop_threshold > 1.0)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Drop threshold should be a value between or "
+ "equal to 0 and 1.0 but got %0.3f",
+ drop_threshold);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ double old = siridb->drop_threshold;
+ siridb->drop_threshold = drop_threshold;
+ if (siridb_save(siridb))
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error while saving database changes!");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+
+ log_info(
+ MSG_SUCCESS_SET_DROP_THRESHOLD,
+ old,
+ siridb->drop_threshold);
+
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCESS_SET_DROP_THRESHOLD,
+ old,
+ siridb->drop_threshold);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+ }
+}
+
+static void exit_set_expiration_xxx(
+ uv_async_t * handle,
+ uint64_t * expirep,
+ uint8_t tp)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+ MASTER_CHECK_VERSION(siridb, "2.0.35")
+
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+ uint64_t expiration = (uint64_t) CLERI_NODE_DATA(node);
+
+ if (IS_MASTER && expiration)
+ {
+ struct timespec now;
+ clock_gettime(CLOCK_REALTIME, &now);
+ now.tv_sec -= (3600*24); /* remove one dat to be save */
+ uint64_t now_ts = siridb_time_now(siridb, now);
+
+ if (expiration >= now_ts)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Shard expiration time should be a value greater "
+ "than or equal to zero (0) and smaller "
+ "than %"PRIu64" but got %" PRId64,
+ now_ts, (int64_t) CLERI_NODE_DATA(node));
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ query_wrapper_t * q_wrapper = (query_wrapper_t *) query->data;
+ double percent = siridb_shards_count_percent(
+ siridb,
+ now_ts - expiration,
+ tp);
+
+ if ((~q_wrapper->flags & QUERIES_IGNORE_DROP_THRESHOLD) &&
+ percent >= siridb->drop_threshold)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "This query would drop %0.2f%% of the shards in pool %u. "
+ "Add \'set ignore_threshold true\' to the query "
+ "statement if you really want to do this.",
+ percent * 100,
+ siridb->server->pool);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ }
+
+ uint64_t old = *expirep;
+ *expirep = expiration;
+
+ siridb_update_shard_expiration(siridb);
+
+ if (siridb_save(siridb))
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error while saving database changes!");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+
+ log_info(
+ MSG_SUCCESS_SET_EXPIRATION,
+ old,
+ *expirep);
+
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCESS_SET_EXPIRATION,
+ old,
+ *expirep);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+static void exit_set_expiration_log(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ exit_set_expiration_xxx(
+ handle,
+ &siridb->expiration_log,
+ SIRIDB_SHARD_TP_LOG);
+}
+
+
+static void exit_set_expiration_num(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ exit_set_expiration_xxx(
+ handle,
+ &siridb->expiration_num,
+ SIRIDB_SHARD_TP_NUMBER);
+}
+
+static void exit_set_list_limit(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+ MASTER_CHECK_VERSION(siridb, "2.0.17")
+
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+
+ uint64_t limit = xstr_to_uint64(node->str, node->len);
+
+ if (limit < 1000 || limit >= 4294967296)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "List limit should be a value greater than or equal to 1000 "
+ "and smaller than 4294967296 but got %" PRIu64,
+ limit);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ uint32_t old = siridb->list_limit;
+ siridb->list_limit = (uint32_t) limit;
+
+ if (siridb_save(siridb))
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error while saving database changes!");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+
+ log_info(
+ MSG_SUCCESS_SET_LIST_LIMIT,
+ old,
+ siridb->list_limit);
+
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCESS_SET_LIST_LIMIT,
+ old,
+ siridb->list_limit);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+ }
+}
+
+static void exit_set_log_level(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ assert (query->data != NULL);
+
+ cleri_node_t * node =
+ query->nodes->node->children->next->next->node->children->node;
+
+ int log_level;
+
+ switch (node->cl_obj->gid)
+ {
+ case CLERI_GID_K_DEBUG:
+ log_level = LOGGER_DEBUG;
+ break;
+ case CLERI_GID_K_INFO:
+ log_level = LOGGER_INFO;
+ break;
+ case CLERI_GID_K_WARNING:
+ log_level = LOGGER_WARNING;
+ break;
+ case CLERI_GID_K_ERROR:
+ log_level = LOGGER_ERROR;
+ break;
+ case CLERI_GID_K_CRITICAL:
+ log_level = LOGGER_CRITICAL;
+ break;
+ default:
+ assert (0);
+ log_level = LOGGER_DEBUG;
+ break;
+ }
+
+ if (q_alter->alter_tp == QUERY_ALTER_SERVERS)
+ {
+ /*
+ * alter_servers
+ */
+ cexpr_t * where_expr = ((query_list_t *) query->data)->where_expr;
+ siridb_server_walker_t wserver = {
+ .server=siridb->server,
+ .siridb=siridb
+ };
+
+ if (where_expr == NULL || cexpr_run(
+ where_expr,
+ (cexpr_cb_t) siridb_server_cexpr_cb,
+ &wserver))
+ {
+ logger_set_level(log_level);
+ q_alter->n++;
+ }
+
+ if (IS_MASTER)
+ {
+ /*
+ * Hide log level information so we can later create an appropriate
+ * message.
+ */
+ q_alter->n += log_level << 16;
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_alter_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_raw(query->packer, (const unsigned char *) "servers", 7);
+ qp_add_int64(query->packer, q_alter->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+ else
+ {
+ /*
+ * alter_server
+ *
+ * we can set the success message, we just ignore the message in case
+ * an error occurs.
+ */
+ siridb_server_t * server = q_alter->via.server;
+
+ QP_ADD_SUCCESS
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCES_SET_LOG_LEVEL,
+ logger_level_name(log_level),
+ server->name);
+
+ if (server == siridb->server)
+ {
+ logger_set_level(log_level);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ else
+ {
+ QP_PACK_INT16(buffer, log_level)
+
+ if (siridb_server_is_online(server))
+ {
+ sirinet_pkg_t * pkg = sirinet_pkg_new(
+ 0,
+ 3,
+ BPROTO_LOG_LEVEL_UPDATE,
+ buffer);
+ if (pkg != NULL)
+ {
+ /* handle will be bound to a timer so we should increment */
+ siri_async_incref(handle);
+ if (siridb_server_send_pkg(
+ server,
+ pkg,
+ 0,
+ (sirinet_promise_cb) on_ack_response,
+ handle,
+ 0))
+ {
+ /*
+ * signal is raised and 'on_ack_response' will not be
+ * called
+ */
+ free(pkg);
+ siri_async_decref(&handle);
+ }
+ }
+ }
+ else
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot set log level, '%s' is currently unavailable",
+ server->name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ }
+ }
+}
+
+static void exit_set_port(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_server_t * server = ((query_alter_t *) query->data)->via.server;
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+ siridb_t * siridb = query->client->siridb;
+
+ if (siridb->server == server || server->client != NULL)
+ {
+ sprintf(query->err_msg, MSG_ERR_SERVER_ADDRESS);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+
+ uint64_t port = xstr_to_uint64(node->str, node->len);
+
+ if (port > 65535)
+ {
+ sprintf(query->err_msg,
+ "Server port must be a value between 0 and 65535");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ int rc;
+ rc = siridb_server_update_address(siridb, server, server->address, port);
+ switch (rc)
+ {
+ case -1:
+ sprintf(query->err_msg, "Error while updating server address");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ break;
+
+ case 0:
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Server port is already set to '%u'",
+ server->port);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ break;
+
+ case 1:
+ QP_ADD_SUCCESS
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCESS_SET_ADDR_PORT,
+ server->name);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ break;
+ }
+ }
+}
+
+static void exit_set_select_points_limit(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+ MASTER_CHECK_VERSION(siridb, "2.0.17")
+
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+
+ uint64_t limit = xstr_to_uint64(node->str, node->len);
+
+ if (limit < 1 || limit >= 4294967296)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Select points limit should be a value greater than 0 "
+ "and smaller than 4294967296 but got %" PRIu64,
+ limit);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ uint32_t old = siridb->select_points_limit;
+ siridb->select_points_limit = (uint32_t) limit;
+
+ if (siridb_save(siridb))
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error while saving database changes!");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+
+ log_info(
+ MSG_SUCCESS_SET_SELECT_POINTS_LIMIT,
+ old,
+ siridb->select_points_limit);
+
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCESS_SET_SELECT_POINTS_LIMIT,
+ old,
+ siridb->select_points_limit);
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+ }
+}
+
+static void exit_set_tee_pipe_name(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ assert (query->data != NULL);
+
+ cleri_node_t * node =
+ query->nodes->node->children->next->next->node->children->node;
+
+ char pipe_name[node->len - 1];
+ char * p_pipe_name = NULL;
+
+ if (node->cl_obj->gid == CLERI_GID_STRING)
+ {
+ xstr_extract_string(pipe_name, node->str, node->len);
+ p_pipe_name = pipe_name;
+ }
+
+ if (q_alter->alter_tp == QUERY_ALTER_SERVERS)
+ {
+ /*
+ * alter_servers
+ */
+ cexpr_t * where_expr = ((query_list_t *) query->data)->where_expr;
+ siridb_server_walker_t wserver = {
+ .server=siridb->server,
+ .siridb=siridb
+ };
+
+ if (where_expr == NULL || cexpr_run(
+ where_expr,
+ (cexpr_cb_t) siridb_server_cexpr_cb,
+ &wserver))
+ {
+ (void) siridb_tee_set_pipe_name(siridb->tee, p_pipe_name);
+ if (siridb_save(siridb))
+ {
+ log_critical("Could not save database changes (database: '%s')",
+ siridb->dbname);
+ }
+ q_alter->n++;
+ }
+
+ if (IS_MASTER)
+ {
+ /*
+ * This is a trick because we share with setting log level on
+ * multiple servers at once.
+ */
+ q_alter->n += LOGGER_NUM_LEVELS << 16;
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_SERVERS,
+ (sirinet_promises_cb) on_alter_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_raw(query->packer, (const unsigned char *) "servers", 7);
+ qp_add_int64(query->packer, q_alter->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+ else
+ {
+ /*
+ * alter_server
+ *
+ * we can set the success message, we just ignore the message in case
+ * an error occurs.
+ */
+ siridb_server_t * server = q_alter->via.server;
+
+ QP_ADD_SUCCESS
+ qp_add_fmt_safe(query->packer,
+ MSG_SUCCES_SET_TEE_PIPE_NAME,
+ p_pipe_name ? p_pipe_name : "disabled",
+ server->name);
+
+ if (server == siridb->server)
+ {
+ (void) siridb_tee_set_pipe_name(siridb->tee, p_pipe_name);
+ if (siridb_save(siridb))
+ {
+ log_critical("Could not save database changes (database: '%s')",
+ siridb->dbname);
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ else
+ {
+
+ if (siridb_server_is_online(server))
+ {
+ sirinet_pkg_t * pkg = sirinet_pkg_new(
+ 0,
+ p_pipe_name ? strlen(p_pipe_name) : 0,
+ BPROTO_TEE_PIPE_NAME_UPDATE,
+ (unsigned char *) p_pipe_name);
+ if (pkg != NULL)
+ {
+ /* handle will be bound to a timer so we should increment */
+ siri_async_incref(handle);
+ if (siridb_server_send_pkg(
+ server,
+ pkg,
+ 0,
+ (sirinet_promise_cb) on_ack_response,
+ handle,
+ 0))
+ {
+ /*
+ * signal is raised and 'on_ack_response' will not be
+ * called
+ */
+ free(pkg);
+ siri_async_decref(&handle);
+ }
+ }
+ }
+ else
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot set pipe name, '%s' is currently unavailable",
+ server->name);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ }
+ }
+}
+
+static void exit_set_timezone(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ cleri_node_t * node = query->nodes->node->children->next->next->node;
+ siridb_t * siridb = query->client->siridb;
+
+ MASTER_CHECK_ACCESSIBLE(siridb)
+
+ char timezone[node->len - 1];
+ xstr_extract_string(timezone, node->str, node->len);
+
+ iso8601_tz_t new_tz = iso8601_tz(timezone);
+
+ if (new_tz < 0)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Unknown time zone: '%s'. (see 'help timezones' "
+ "for a list of valid time zones)",
+ timezone);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+
+ }
+ else if (siridb->tz == new_tz)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Database '%s' is already set to time-zone '%s'.",
+ siridb->dbname,
+ timezone);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ QP_ADD_SUCCESS
+
+ qp_add_fmt_safe(
+ query->packer,
+ MSG_SUCCES_SET_TIMEZONE,
+ iso8601_tzname(siridb->tz),
+ iso8601_tzname(new_tz));
+
+ siridb->tz = new_tz;
+
+ if (siridb_save(siridb))
+ {
+ log_critical("Could not save database changes (database: '%s')",
+ siridb->dbname);
+ }
+
+ if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_update_xxx_response,
+ 0);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+}
+
+static void exit_show_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_user_t * db_user = query->client->origin;
+ SIRIPARSER_MASTER_CHECK_ACCESS(db_user, SIRIDB_ACCESS_SHOW)
+
+ cleri_children_t * children =
+ query->nodes->node->children->next->node->children;
+ siridb_props_cb prop_cb;
+
+ assert (query->packer == NULL);
+
+ query->packer = sirinet_packer_new(4096);
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+ qp_add_raw(query->packer, (const unsigned char *) "data", 4);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ /* set props.h (who_am_i) to current db_name */
+ props_set_who_am_i(db_user->name);
+
+ if (children == NULL || children->node == NULL)
+ {
+ /* show all properties */
+ int i;
+
+ for (i = 0; i < KW_COUNT; i++)
+ {
+ if ((prop_cb = props_get_cb(i)) == NULL)
+ {
+ continue;
+ }
+ prop_cb(siridb, query->packer, 1);
+ }
+ }
+ else
+ {
+ /* show selected properties chosen by query */
+ while (1)
+ {
+ /* get the callback */
+ prop_cb = props_get_cb(children->node->children->node->
+ cl_obj->gid - KW_OFFSET);
+ assert (prop_cb != NULL); /* all props are implemented */
+ prop_cb(siridb, query->packer, 1);
+
+ if (children->next == NULL)
+ {
+ break;
+ }
+
+ /* skip one which is the delimiter */
+ children = children->next->next;
+ }
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_tag_series(uv_async_t * handle)
+{
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+
+ if (IS_MASTER)
+ {
+ qp_add_fmt_safe(
+ query->packer,
+ MSG_SUCCESS_TAG,
+ ((query_alter_t *) query->data)->n);
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_timeit_stmt(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ struct timespec end;
+ char * name = siridb->server->name;
+ clock_gettime(CLOCK_REALTIME, &end);
+
+ qp_add_type(query->timeit, QP_MAP2);
+ qp_add_raw(query->timeit, (const unsigned char *) "server", 6);
+ qp_add_string(query->timeit, name);
+ qp_add_raw(query->timeit, (const unsigned char *) "time", 4);
+ qp_add_double(query->timeit,
+ (double) (end.tv_sec - query->start.tv_sec) +
+ (double) (end.tv_nsec - query->start.tv_nsec) / 1000000000.0f);
+
+ if (query->packer == NULL)
+ {
+ /* lets give the new packer the exact size so we do not
+ * need a realloc */
+ query->packer = sirinet_packer_new(
+ query->timeit->len +
+ 1 +
+ sizeof(sirinet_pkg_t));
+
+ if (query->packer == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ qp_add_type(query->packer, QP_MAP_OPEN);
+ }
+
+ /* extend packer with timeit information */
+ qp_packer_extend(query->packer, query->timeit);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void exit_untag_series(uv_async_t * handle)
+{
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+
+ if (IS_MASTER)
+ {
+ qp_add_fmt_safe(
+ query->packer,
+ MSG_SUCCESS_UNTAG,
+ ((query_alter_t *) query->data)->n);
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+/******************************************************************************
+ * Async loop functions.
+ *****************************************************************************/
+
+static void async_count_series(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_series_t * series;
+ query_count_t * q_count = (query_count_t *) query->data;
+ uint8_t async_more = 0;
+
+ size_t index_end = q_count->vec_index + MAX_ITERATE_COUNT;
+
+ if (index_end >= q_count->vec->len)
+ {
+ index_end = q_count->vec->len;
+ }
+ else
+ {
+ async_more = 1;
+ }
+
+ for (; q_count->vec_index < index_end; q_count->vec_index++)
+ {
+ series = (siridb_series_t *) q_count->vec->data[q_count->vec_index];
+ q_count->n += cexpr_run(
+ q_count->where_expr,
+ (cexpr_cb_t) siridb_series_cexpr_cb,
+ series);
+ siridb_series_decref(series);
+ }
+
+ if (async_more)
+ {
+ uv_async_send(handle);
+ }
+ else if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_POOLS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void async_count_series_length(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ siridb_series_t * series;
+ query_count_t * q_count = (query_count_t *) query->data;
+ uint8_t async_more = 0;
+
+ size_t index_end = q_count->vec_index + MAX_ITERATE_COUNT;
+
+ if (index_end >= q_count->vec->len)
+ {
+ index_end = q_count->vec->len;
+ }
+ else
+ {
+ async_more = 1;
+ }
+
+ for (; q_count->vec_index < index_end; q_count->vec_index++)
+ {
+ series = (siridb_series_t *) q_count->vec->data[q_count->vec_index];
+
+ if (cexpr_run(
+ q_count->where_expr,
+ (cexpr_cb_t) siridb_series_cexpr_cb,
+ series))
+ {
+ q_count->n += series->length;
+ }
+
+ siridb_series_decref(series);
+ }
+
+ if (async_more)
+ {
+ uv_async_send(handle);
+ }
+ else if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_POOLS,
+ (sirinet_promises_cb) on_count_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void async_drop_series(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_drop_t * q_drop = (query_drop_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ siridb_series_t * series;
+ uint8_t async_more = 0;
+
+ size_t index_end = q_drop->vec_index + MAX_ITERATE_COUNT;
+
+ if (index_end >= q_drop->vec->len)
+ {
+ index_end = q_drop->vec->len;
+ }
+ else
+ {
+ async_more = 1;
+ }
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ for (; q_drop->vec_index < index_end; q_drop->vec_index++)
+ {
+ series = (siridb_series_t *) q_drop->vec->data[q_drop->vec_index];
+ siridb_series_drop(siridb, series);
+ siridb_series_decref(series);
+ }
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ /* flush dropped file change to disk */
+ if (q_drop->vec->len)
+ {
+ siridb_series_flush_dropped(siridb);
+ }
+
+ if (async_more)
+ {
+ uv_async_send(handle);
+ }
+ else if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_drop_series_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_drop->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void async_drop_shards(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_drop_t * q_drop = (query_drop_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ if (q_drop->shards_list->len)
+ {
+ siridb_shard_t * shard = (siridb_shard_t *) vec_pop(
+ q_drop->shards_list);
+
+ siridb_shard_drop(
+ shard,
+ siridb);
+ siridb_shard_decref(shard);
+ }
+
+ if (q_drop->shards_list->len)
+ {
+ uv_async_send(handle);
+ }
+ else if (IS_MASTER)
+ {
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_UPDATE,
+ (sirinet_promises_cb) on_drop_shards_response,
+ 0);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_drop->n);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void async_filter_series(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_wrapper_t * q_wrapper = query->data;
+ cexpr_t * where_expr = q_wrapper->where_expr;
+ uint8_t async_more = 0;
+ siridb_series_t * series;
+ size_t index_end = q_wrapper->vec_index + MAX_ITERATE_COUNT;
+
+ if (index_end >= q_wrapper->vec->len)
+ {
+ index_end = q_wrapper->vec->len;
+ }
+ else
+ {
+ async_more = 1;
+ }
+
+ for (; q_wrapper->vec_index < index_end; q_wrapper->vec_index++)
+ {
+ series = (siridb_series_t *)
+ q_wrapper->vec->data[q_wrapper->vec_index];
+
+ if (cexpr_run(
+ where_expr,
+ (cexpr_cb_t) siridb_series_cexpr_cb,
+ series))
+ {
+ if (imap_add(q_wrapper->series_map, series->id, series))
+ {
+ log_critical("Cannot add filtered series to internal map.");
+ siridb_series_decref(series);
+ }
+ }
+ else
+ {
+ siridb_series_decref(series);
+ }
+ }
+
+ if (async_more)
+ {
+ uv_async_send(handle);
+ }
+ else
+ {
+ /* free the s-list object and reset index */
+ vec_free(q_wrapper->vec);
+
+ q_wrapper->vec = NULL;
+ q_wrapper->vec_index = 0;
+
+ /* cleanup where statement since we do not need it anymore */
+ cexpr_free(q_wrapper->where_expr);
+ q_wrapper->where_expr = NULL;
+
+ /* we now processed the where statement, continue... */
+ switch (q_wrapper->tp)
+ {
+ case QUERIES_DROP:
+ exit_drop_series(handle);
+ break;
+ case QUERIES_SELECT:
+ exit_select_aggregate(handle);
+ break;
+ default:
+ assert (0);
+ break;
+ }
+ }
+}
+
+static void async_list_series(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_list_t * q_list = (query_list_t *) query->data;
+ vec_t * props = q_list->props;
+ cexpr_t * where_expr = q_list->where_expr;
+ uint8_t async_more = 0;
+ siridb_series_t * series;
+ size_t i;
+ size_t index_end = q_list->vec_index + MAX_ITERATE_COUNT;
+
+ if (index_end >= q_list->vec->len)
+ {
+ index_end = q_list->vec->len;
+ }
+ else
+ {
+ async_more = 1;
+ }
+
+ for (; q_list->limit && q_list->vec_index < index_end;
+ q_list->vec_index++)
+ {
+ series = (siridb_series_t *) q_list->vec->data[q_list->vec_index];
+
+ if (where_expr == NULL || cexpr_run(
+ where_expr,
+ (cexpr_cb_t) siridb_series_cexpr_cb,
+ series))
+ {
+ if (!--q_list->limit)
+ {
+ async_more = 0;
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ for (i = 0; i < props->len; i++)
+ {
+ switch(*((uint32_t *) props->data[i]))
+ {
+ case CLERI_GID_K_NAME:
+ qp_add_raw(
+ query->packer,
+ (const unsigned char *) series->name,
+ series->name_len);
+ break;
+ case CLERI_GID_K_LENGTH:
+ qp_add_int64(query->packer, (int64_t) series->length);
+ break;
+ case CLERI_GID_K_TYPE:
+ qp_add_string(query->packer, series_type_map[series->tp]);
+ break;
+ case CLERI_GID_K_POOL:
+ qp_add_int64(query->packer, (int64_t) series->pool);
+ break;
+ case CLERI_GID_K_SHARD_DURATION:
+ qp_add_int64(query->packer, (int64_t) (series->idx
+ ? series->idx->shard->duration : 0));
+ break;
+ case CLERI_GID_K_START:
+ qp_add_int64(query->packer, (int64_t) series->start);
+ break;
+ case CLERI_GID_K_END:
+ qp_add_int64(query->packer, (int64_t) series->end);
+ break;
+ }
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+ }
+
+ siridb_series_decref(series);
+ }
+
+ if (async_more)
+ {
+ uv_async_send(handle);
+ }
+ else if (IS_MASTER && q_list->limit)
+ {
+ /* we have not reached the limit, send the query to other pools */
+ siridb_query_forward(
+ handle,
+ SIRIDB_QUERY_FWD_POOLS,
+ (sirinet_promises_cb) on_list_xxx_response,
+ 0);
+ }
+ else
+ {
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void async_no_points_aggregate(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_select_t * q_select = query->data;
+ siridb_t * siridb = query->client->siridb;
+ uint8_t async_more = 0;
+ siridb_series_t * series;
+ siridb_points_t * points;
+ siridb_points_t * aggr_points;
+ int required_shard = 0;
+
+ for (; q_select->vec_index < q_select->vec->len;
+ ++q_select->vec_index)
+ {
+ const char * name;
+ size_t i;
+
+ if (required_shard > MAX_BATCH_REQUIRE_SHARD)
+ {
+ async_more = 1;
+ break;
+ }
+
+ series = (siridb_series_t *)
+ q_select->vec->data[q_select->vec_index];
+ /*
+ * We must decrement the ref count immediately since the index is
+ * incremented by one. The series will not be freed since at least
+ * 'series_map' still has a reference.
+ */
+ siridb_series_decref(series);
+
+ assert (q_select->alist->len >= 1);
+
+ siridb_aggr_t * aggr = q_select->alist->data[0];
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ switch (aggr->gid)
+ {
+ case CLERI_GID_F_COUNT:
+ points = siridb_series_get_count(series);
+ break;
+ case CLERI_GID_F_FIRST:
+ points = siridb_series_get_first(series, &required_shard);
+ break;
+ case CLERI_GID_F_LAST:
+ points = siridb_series_get_last(series, &required_shard);
+ break;
+ default:
+ assert (0);
+ points = NULL;
+ }
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (points == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ for (i = 1; points->len && i < q_select->alist->len; i++)
+ {
+ aggr_points = siridb_aggregate_run(
+ points,
+ (siridb_aggr_t *) q_select->alist->data[i],
+ query->err_msg);
+
+ if (aggr_points != points)
+ {
+ siridb_points_free(points);
+ }
+
+ if (aggr_points == NULL)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ points = aggr_points;
+ }
+
+ q_select->n += points->len;
+
+ if (q_select->merge_as == NULL)
+ {
+ name = siridb_presuf_name(
+ q_select->presuf,
+ series->name,
+ series->name_len);
+
+ if (name == NULL || ct_add(q_select->result, name, points))
+ {
+ sprintf(query->err_msg, "Error adding points to map.");
+ siridb_points_free(points);
+ log_critical("Critical error adding points");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ }
+ else
+ {
+ vec_t ** plist;
+
+ name = siridb_presuf_name(
+ q_select->presuf,
+ q_select->merge_as,
+ strlen(q_select->merge_as));
+
+ plist = (vec_t **) ct_getaddr(q_select->result, name);
+
+ if ( name == NULL ||
+ plist == NULL ||
+ vec_append_safe(plist, points))
+ {
+ sprintf(query->err_msg, "Error adding points to map.");
+ siridb_points_free(points);
+ log_critical("Critical error adding points");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ }
+ }
+
+ if (async_more)
+ {
+ uv_async_send(handle);
+ }
+ else
+ {
+ siridb_aggregate_list_free(q_select->alist);
+ q_select->alist = NULL;
+
+ vec_free(q_select->vec);
+ q_select->vec = NULL;
+ q_select->vec_index = 0;
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void async_select_aggregate(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_select_t * q_select = query->data;
+ siridb_t * siridb = query->client->siridb;
+ uint8_t async_more = 0;
+ siridb_series_t * series;
+ siridb_points_t * points;
+ siridb_points_t * aggr_points;
+
+ if (q_select->n > siridb->select_points_limit)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Query has reached the maximum number of selected points "
+ "(%u). Please use another time window, an aggregation "
+ "function or select less series to reduce the number of "
+ "points.",
+ siridb->select_points_limit);
+
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ series = (siridb_series_t *)
+ q_select->vec->data[q_select->vec_index];
+
+ /*
+ * We must decrement the ref count immediately since we now update the
+ * index by one. The series will not be freed since at least 'series_map'
+ * still has a reference.
+ */
+ siridb_series_decref(series);
+
+ if ((++q_select->vec_index) < q_select->vec->len)
+ {
+ async_more = 1;
+ }
+
+ /* We try to read the points from the cache in case a cache is created.
+ * If there are more select functions left we create a copy of the cache.
+ * When this is the last select function we pop from the cache since the
+ * points are no longer required.
+ */
+ points = (q_select->points_map == NULL) ?
+ NULL :
+ q_select->nselects ?
+ siridb_points_copy(imap_get(q_select->points_map, series->id)):
+ imap_pop(q_select->points_map, series->id);
+
+ if (points == NULL)
+ {
+ uv_mutex_lock(&siridb->series_mutex);
+
+ points = (series->flags & SIRIDB_SERIES_IS_DROPPED) ?
+ NULL : siridb_series_get_points(
+ series,
+ q_select->start_ts,
+ q_select->end_ts);
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ /* when having a cache and points, add a copy of points to the cache */
+ if (q_select->points_map != NULL && points != NULL)
+ {
+ siridb_points_t * cpoints = siridb_points_copy(points);
+ if (cpoints != NULL &&
+ imap_add(q_select->points_map, series->id, cpoints))
+ {
+ siridb_points_free(cpoints);
+ }
+ }
+ }
+
+ if (points != NULL)
+ {
+ const char * name;
+ size_t i;
+
+ for (i = 0; points->len && i < q_select->alist->len; i++)
+ {
+ aggr_points = siridb_aggregate_run(
+ points,
+ (siridb_aggr_t *) q_select->alist->data[i],
+ query->err_msg);
+
+ if (aggr_points != points)
+ {
+ siridb_points_free(points);
+ }
+
+ if (aggr_points == NULL)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ points = aggr_points;
+ }
+
+ q_select->n += points->len;
+
+ if (q_select->merge_as == NULL)
+ {
+ name = siridb_presuf_name(
+ q_select->presuf,
+ series->name,
+ series->name_len);
+
+ if (name == NULL || ct_add(q_select->result, name, points))
+ {
+ sprintf(query->err_msg, "Error adding points to map.");
+ siridb_points_free(points);
+ log_critical("Critical error adding points");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ }
+ else
+ {
+ vec_t ** plist;
+
+ name = siridb_presuf_name(
+ q_select->presuf,
+ q_select->merge_as,
+ strlen(q_select->merge_as));
+
+ plist = (vec_t **) ct_getaddr(q_select->result, name);
+
+ if ( name == NULL ||
+ plist == NULL ||
+ vec_append_safe(plist, points))
+ {
+ sprintf(query->err_msg, "Error adding points to map.");
+ siridb_points_free(points);
+ log_critical("Critical error adding points");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+ }
+ }
+
+ if (async_more)
+ {
+ uv_async_send(handle);
+ }
+ else
+ {
+ siridb_aggregate_list_free(q_select->alist);
+ q_select->alist = NULL;
+
+ vec_free(q_select->vec);
+ q_select->vec = NULL;
+ q_select->vec_index = 0;
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+static void async_series_re(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_wrapper_t * q_wrapper = query->data;
+ uint8_t async_more = 0;
+ siridb_series_t * series;
+ size_t index_end = q_wrapper->vec_index + MAX_ITERATE_COUNT;
+
+ if (index_end >= q_wrapper->vec->len)
+ {
+ index_end = q_wrapper->vec->len;
+ }
+ else
+ {
+ async_more = 1;
+ }
+
+ int pcre_exec_ret;
+
+ for (; q_wrapper->vec_index < index_end; q_wrapper->vec_index++)
+ {
+ series = (siridb_series_t *)
+ q_wrapper->vec->data[q_wrapper->vec_index];
+
+ pcre_exec_ret = pcre2_match(
+ q_wrapper->regex,
+ (PCRE2_SPTR8) series->name,
+ series->name_len,
+ 0, /* start looking at this point */
+ 0, /* OPTIONS */
+ q_wrapper->match_data,
+ 0); /* length of sub_str_vec */
+ if ( pcre_exec_ret < 0 ||
+ imap_add(q_wrapper->series_tmp, series->id, series))
+ {
+ siridb_series_decref(series);
+ }
+ }
+
+ if (async_more)
+ {
+ uv_async_send(handle);
+ }
+ else
+ {
+ /* free the s-list object and reset index */
+ vec_free(q_wrapper->vec);
+
+ pcre2_code_free(q_wrapper->regex);
+ pcre2_match_data_free(q_wrapper->match_data);
+
+ q_wrapper->regex = NULL;
+ q_wrapper->match_data = NULL;
+
+ q_wrapper->vec = NULL;
+ q_wrapper->vec_index = 0;
+
+ if (q_wrapper->update_cb != NULL)
+ {
+ (*q_wrapper->update_cb)(
+ q_wrapper->series_map,
+ q_wrapper->series_tmp,
+ (imap_free_cb) &siridb__series_decref);
+ }
+ q_wrapper->series_tmp = NULL;
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+/******************************************************************************
+ * On Response functions
+ *****************************************************************************/
+
+/*
+ * Call-back function: sirinet_promise_cb
+ */
+static void on_ack_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ uv_async_t * handle = (uv_async_t *) promise->data;
+
+ /* decrement the handle reference counter */
+ siri_async_decref(&handle);
+
+ if (handle != NULL)
+ {
+ siridb_query_t * query = handle->data;
+
+ if (status == PROMISE_SUCCESS)
+ {
+ switch (pkg->tp)
+ {
+ case BPROTO_ACK_LOG_LEVEL:
+ /* success message is already set */
+ break;
+ case BPROTO_ACK_ENABLE_BACKUP_MODE:
+ /* success message is already set */
+ break;
+ case BPROTO_ACK_DISABLE_BACKUP_MODE:
+ /* success message is already set */
+ break;
+ case BPROTO_ACK_TEE_PIPE_NAME:
+ /* success message is already set */
+ break;
+ default:
+ status = PROMISE_PKG_TYPE_ERROR;
+ break;
+ }
+ }
+
+ if (status)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error occurred while sending the request to '%s' (%s)",
+ promise->server->name,
+ sirinet_promise_strstatus(status));
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+ }
+
+ /* we must free the promise */
+ sirinet_promise_decref(promise);
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+static void on_alter_xxx_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ siridb_query_t * query = handle->data;
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ qp_obj_t qp_count;
+ query_alter_t * q_alter = (query_alter_t *) query->data;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ continue;
+ }
+
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg != NULL && pkg->tp == BPROTO_RES_QUERY)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && /* servers etc.*/
+ qp_is_int(qp_next(&unpacker, &qp_count))) /* one result*/
+ {
+ q_alter->n += qp_count.via.int64;
+
+ /* extract time-it info if needed */
+ if (query->timeit != NULL)
+ {
+ siridb_query_timeit_from_unpacker(query, &unpacker);
+ }
+ }
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+ /*
+ * Note: since this function has the sole purpose for alter servers
+ * and setting log levels or pipe name, we now simply add the
+ * message here.
+ */
+ QP_ADD_SUCCESS
+
+ if ((q_alter->n >> 16) >= LOGGER_NUM_LEVELS)
+ {
+ log_info(MSG_SUCCES_SET_TEE_PIPE_NAME_MULTI, q_alter->n & 0xffff);
+ qp_add_fmt_safe(
+ query->packer,
+ MSG_SUCCES_SET_TEE_PIPE_NAME_MULTI,
+ q_alter->n & 0xffff);
+ }
+ else
+ {
+ log_info(MSG_SUCCES_SET_LOG_LEVEL_MULTI,
+ logger_level_name(q_alter->n >> 16),
+ q_alter->n & 0xffff);
+
+ qp_add_fmt_safe(
+ query->packer,
+ MSG_SUCCES_SET_LOG_LEVEL_MULTI,
+ logger_level_name(q_alter->n >> 16),
+ q_alter->n & 0xffff);
+ }
+
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+static void on_count_xxx_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ uint8_t error_tp = 0;
+ siridb_query_t * query = handle->data;
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ qp_obj_t qp_count;
+ query_count_t * q_count = (query_count_t *) query->data;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ continue;
+ }
+
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg != NULL && pkg->tp == BPROTO_RES_QUERY)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && /* servers etc.*/
+ qp_is_int(qp_next(&unpacker, &qp_count))) /* one result*/
+ {
+ q_count->n += qp_count.via.int64;
+
+ /* extract time-it info if needed */
+ if (query->timeit != NULL)
+ {
+ siridb_query_timeit_from_unpacker(query, &unpacker);
+ }
+ }
+ }
+ else if (pkg != NULL &&
+ !error_tp &&
+ sirinet_protocol_is_error_msg(pkg->tp) &&
+ siridb_query_err_from_pkg(query, pkg) == 0)
+ {
+ error_tp = pkg->tp;
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+
+ if (error_tp)
+ {
+ siridb_query_send_error(handle, error_tp);
+ }
+ else
+ {
+ qp_add_int64(query->packer, q_count->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+static void on_drop_series_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ siridb_query_t * query = handle->data;
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ qp_obj_t qp_drop;
+ query_drop_t * q_drop = (query_drop_t *) query->data;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ continue;
+ }
+
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg != NULL && pkg->tp == BPROTO_RES_QUERY)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && /* servers etc.*/
+ qp_is_int(qp_next(&unpacker, &qp_drop))) /* one result */
+ {
+ q_drop->n += qp_drop.via.int64;
+
+ /* extract time-it info if needed */
+ if (query->timeit != NULL)
+ {
+ siridb_query_timeit_from_unpacker(query, &unpacker);
+ }
+ }
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+
+ qp_add_fmt(query->packer, MSG_SUCCES_DROP_SERIES, q_drop->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+static void on_drop_shards_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ siridb_query_t * query = handle->data;
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ qp_obj_t qp_drop;
+ query_drop_t * q_drop = (query_drop_t *) query->data;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ continue;
+ }
+
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg != NULL && pkg->tp == BPROTO_RES_QUERY)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && /* shards */
+ qp_is_int(qp_next(&unpacker, &qp_drop))) /* one result */
+ {
+ q_drop->n += qp_drop.via.int64;
+
+ /* extract time-it info if needed */
+ if (query->timeit != NULL)
+ {
+ siridb_query_timeit_from_unpacker(query, &unpacker);
+ }
+ }
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+
+ qp_add_fmt(query->packer, MSG_SUCCES_DROP_SHARDS, q_drop->n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+static void on_groups_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_group_t * group;
+ qp_obj_t qp_name;
+ qp_obj_t qp_series;
+ size_t i;
+
+ siridb_groups_init_nseries(siridb->groups);
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ continue;
+ }
+
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg != NULL && pkg->tp == BPROTO_RES_GROUPS)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_array(qp_next(&unpacker, NULL)))
+ {
+ while ( qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, &qp_name)) &&
+ qp_is_raw_term(&qp_name) &&
+ qp_is_int(qp_next(&unpacker, &qp_series)))
+ {
+ group = (siridb_group_t *) ct_get(
+ siridb->groups->groups,
+ (const char *) qp_name.via.raw);
+ if (group != NULL)
+ {
+ group->n += qp_series.via.int64;
+ }
+ }
+ }
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+
+ query->nodes->cb(handle);
+}
+
+static void on_tags_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb_tag_t * tag;
+ qp_obj_t qp_name;
+ qp_obj_t qp_series;
+ size_t i;
+
+ siridb_tags_init_nseries(siridb->tags);
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ continue;
+ }
+
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg != NULL && pkg->tp == BPROTO_RES_TAGS)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_array(qp_next(&unpacker, NULL)))
+ {
+ while ( qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, &qp_name)) &&
+ qp_is_raw_term(&qp_name) &&
+ qp_is_int(qp_next(&unpacker, &qp_series)))
+ {
+ tag = (siridb_tag_t *) ct_get(
+ siridb->tags->tags,
+ (const char *) qp_name.via.raw);
+ if (tag != NULL)
+ {
+ tag->n += qp_series.via.int64;
+ }
+ }
+ }
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+
+ query->nodes->cb(handle);
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+static void on_list_xxx_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ int error_tp = 0;
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ siridb_query_t * query = handle->data;
+ query_list_t * q_list = (query_list_t *) query->data;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ continue;
+ }
+
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg != NULL && pkg->tp == BPROTO_RES_QUERY)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && /* columns */
+ qp_is_array(qp_skip_next(&unpacker)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && /* series/... */
+ qp_is_array(qp_next(&unpacker, NULL))) /* results */
+ {
+ while (qp_is_array(qp_current(&unpacker)))
+ {
+ if (q_list->limit)
+ {
+ qp_packer_extend_fu(query->packer, &unpacker);
+ q_list->limit--;
+ }
+ else
+ {
+ qp_skip_next(&unpacker);
+ }
+ }
+
+ /* extract time-it info if needed */
+ if (query->timeit != NULL)
+ {
+ siridb_query_timeit_from_unpacker(query, &unpacker);
+ }
+ }
+ }
+ else if (pkg != NULL &&
+ !error_tp &&
+ sirinet_protocol_is_error_msg(pkg->tp) &&
+ siridb_query_err_from_pkg(query, pkg) == 0)
+ {
+ error_tp = pkg->tp;
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+
+ if (error_tp)
+ {
+ siridb_query_send_error(handle, error_tp);
+ }
+ else
+ {
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+static void on_select_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ siridb_query_t * query = handle->data;
+ siridb_t * siridb = query->client->siridb;
+ size_t err_count = 0;
+ query_select_t * q_select = query->data;
+ qp_obj_t qp_name;
+ qp_obj_t qp_tp;
+ qp_obj_t qp_len;
+ qp_obj_t qp_points;
+ qp_obj_t qp_err_msg;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ err_count++;
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error occurred while sending the select query to at "
+ "least one required server");
+ }
+ else
+ {
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg == NULL || pkg->tp != BPROTO_RES_QUERY)
+ {
+ err_count++;
+
+ if (pkg != NULL && pkg->tp == BPROTO_ERR_QUERY)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, &qp_err_msg)))
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "%.*s",
+ (int) qp_err_msg.len,
+ qp_err_msg.via.raw);
+ }
+ else
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error occurred while sending request to "
+ "at least '%s'",
+ promise->server->name);
+ }
+ }
+ else
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error occurred while sending request to "
+ "at least '%s'",
+ promise->server->name);
+ }
+ }
+ else
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && /* select */
+ qp_is_map(qp_next(&unpacker, NULL)))
+ {
+ if (q_select->merge_as == NULL)
+ {
+ on_select_unpack_points(
+ &unpacker,
+ q_select,
+ &qp_name,
+ &qp_tp,
+ &qp_len,
+ &qp_points,
+ siridb->select_points_limit);
+ }
+ else
+ {
+ on_select_unpack_merged_points(
+ &unpacker,
+ q_select,
+ &qp_name,
+ &qp_tp,
+ &qp_len,
+ &qp_points,
+ siridb->select_points_limit);
+ }
+
+
+ /* extract time-it info if needed */
+ if (query->timeit != NULL)
+ {
+ siridb_query_timeit_from_unpacker(query, &unpacker);
+ }
+ }
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+ }
+
+ if (q_select->n > siridb->select_points_limit)
+ {
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Query has reached the maximum number of selected points "
+ "(%u). Please use another time window, an aggregation "
+ "function or select less series to reduce the number of "
+ "points.",
+ siridb->select_points_limit);
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else if (err_count)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ uv_async_t * next;
+ uv_work_t * work = malloc(sizeof(uv_work_t));
+ if (work == NULL)
+ {
+ MEM_ERR_RET
+ }
+
+ next = malloc(sizeof(uv_async_t));
+ if (next == NULL)
+ {
+ free(work);
+ MEM_ERR_RET
+ }
+
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+
+ handle = next;
+ handle->data = query;
+ siridb_nodes_next(&query->nodes);
+
+ uv_async_init(
+ siri.loop,
+ handle,
+ (query->nodes == NULL) ?
+ (uv_async_cb) siridb_send_query_result :
+ (uv_async_cb) query->nodes->cb);
+
+ siri_async_incref(handle);
+ work->data = handle;
+ uv_queue_work(
+ siri.loop,
+ work,
+ &master_select_work,
+ &master_select_work_finish);
+
+ }
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+static void on_update_xxx_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ siridb_query_t * query = handle->data;
+ size_t err_count = 0;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ err_count++;
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error occurred while sending the database change to at "
+ "least one required server");
+ }
+ else
+ {
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg == NULL || pkg->tp != BPROTO_RES_QUERY)
+ {
+ err_count++;
+ snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error occurred while sending the database change to "
+ "at least '%s'", promise->server->name);
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+ }
+
+ if (err_count)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ SIRIPARSER_ASYNC_NEXT_NODE
+ }
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle.
+ *
+ * Note: used both for tag and untag response.
+ */
+static void on_tag_response(vec_t * promises, uv_async_t * handle)
+{
+ ON_PROMISES
+
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ qp_unpacker_t unpacker;
+ qp_obj_t qp_tag;
+ size_t i;
+ query_alter_t * q_tag = (query_alter_t *) query->data;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+
+ if (promise == NULL)
+ {
+ continue;
+ }
+
+ pkg = (sirinet_pkg_t *) promise->data;
+
+ if (pkg != NULL && pkg->tp == BPROTO_RES_QUERY)
+ {
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && // success_msg
+ qp_is_int(qp_next(&unpacker, &qp_tag))) // one result
+ {
+ q_tag->n += qp_tag.via.int64;
+
+ /* extract time-it info if needed */
+ if (query->timeit != NULL)
+ {
+ siridb_query_timeit_from_unpacker(query, &unpacker);
+ }
+ }
+ }
+
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+/*
+ * Call-back function: sirinet_promises_cb
+ *
+ * Make sure to run siri_async_incref() on the handle
+ */
+
+
+
+/******************************************************************************
+ * Helper functions
+ *****************************************************************************/
+
+
+static void master_select_work(uv_work_t * work)
+{
+ uv_async_t * handle = (uv_async_t *) work->data;
+ siridb_query_t * query = handle->data;
+ query_select_t * q_select = query->data;
+ siridb_t * siridb = query->client->siridb;
+ siridb->selected_points += q_select->n;
+ int rc = ct_items(
+ q_select->result,
+ (q_select->merge_as == NULL) ?
+ (ct_item_cb) &items_select_master
+ :
+ (ct_item_cb) &items_select_master_merge,
+ handle);
+
+ /* Do not set an error message when rc==1 since in that case the message
+ * is already set.
+ */
+ switch (rc)
+ {
+ case -1:
+ sprintf(query->err_msg, "Memory allocation error.");
+ /* FALLTHRU */
+ /* fall through */
+ case 1:
+ query->flags |= SIRIDB_QUERY_FLAG_ERR;
+ }
+}
+
+static void master_select_work_finish(uv_work_t * work, int status)
+{
+ if (status)
+ {
+ log_error("Select work failed (error: %s)", uv_strerror(status));
+ }
+ else if (!siri_err)
+ {
+ /*
+ * We need to check for SiriDB errors because this task is running in
+ * another thread. In case a siri_err is set, this means we are in forced
+ * closing state and we should not use the handle but let siri close it.
+ */
+
+ uv_async_t * handle = (uv_async_t *) work->data;
+ siridb_query_t * query = handle->data;
+
+ if (query->flags & SIRIDB_QUERY_FLAG_ERR)
+ {
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ }
+ else
+ {
+ uv_async_send(handle);
+ }
+ }
+
+ siri_async_decref((uv_async_t **) &work->data);
+
+ free(work);
+}
+
+static int items_select_master(
+ const char * name,
+ size_t len,
+ siridb_points_t * points,
+ uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ if (query->factor)
+ {
+ siridb_points_ts_correction(points, (double) query->factor);
+ }
+
+ if ( qp_add_raw(query->packer, (const unsigned char *) name, len) ||
+ siridb_points_pack(points, query->packer))
+ {
+ sprintf(query->err_msg, "Memory allocation error.");
+ return -1;
+ }
+
+ return 0;
+}
+
+static int items_select_master_merge(
+ const char * name,
+ size_t len,
+ vec_t * plist,
+ uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_select_t * q_select = query->data;
+ siridb_points_t * points;
+
+ if (qp_add_raw(query->packer, (const unsigned char *) name, len))
+ {
+ sprintf(query->err_msg, "Memory allocation error.");
+ return -1;
+ }
+
+ switch (plist->len)
+ {
+ case 0:
+ points = siridb_points_new(0, TP_INT);
+ if (points == NULL)
+ {
+ sprintf(query->err_msg, "Memory allocation error.");
+ }
+ break;
+ case 1:
+ points = vec_pop(plist);
+ break;
+ default:
+ points = siridb_points_merge(plist, query->err_msg);
+ break;
+ }
+
+ if (q_select->mlist != NULL && points != NULL)
+ {
+ siridb_points_t * aggr_points;
+ size_t i;
+
+ for (i = 0; points->len && i < q_select->mlist->len; i++)
+ {
+ aggr_points = siridb_aggregate_run(
+ points,
+ (siridb_aggr_t *) q_select->mlist->data[i],
+ query->err_msg);
+
+ if (aggr_points != points)
+ {
+ siridb_points_free(points);
+ }
+
+ if (aggr_points == NULL)
+ {
+ return -1; /* (error message is set) */
+ }
+
+ points = aggr_points;
+ }
+ }
+
+ if (points == NULL)
+ {
+ /*
+ * The list will be cleared including the points since 'merge_as'
+ * is still not NULL. (error message is set)
+ */
+ return -1;
+ }
+
+ if (query->factor)
+ {
+ siridb_points_ts_correction(points, (double) query->factor);
+ }
+
+ if (siridb_points_pack(points, query->packer))
+ {
+ sprintf(query->err_msg, "Memory allocation error.");
+ siridb_points_free(points);
+ return -1;
+ }
+
+ siridb_points_free(points);
+
+ return 0;
+}
+
+/*
+ * Returns 0 when successful and -1 in case of an error.
+ * (a SIGNAL is raised in case of an error)
+ */
+static int items_select_other(
+ const char * name,
+ size_t len,
+ siridb_points_t * points,
+ uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ return -(qp_add_raw_term(
+ query->packer, (const unsigned char *) name, len) ||
+ siridb_points_raw_pack(points, query->packer));
+}
+
+/*
+ * Returns 0 when successful and -1 in case of an error.
+ * (a SIGNAL is raised in case of an error)
+ */
+static int items_select_other_merge(
+ const char * name,
+ size_t len,
+ vec_t * plist,
+ uv_async_t * handle)
+{
+ size_t i;
+ siridb_query_t * query = handle->data;
+ int rc = qp_add_raw_term(
+ query->packer, (const unsigned char *) name, len) ||
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ for (i = 0; !rc && i < plist->len; i++)
+ {
+ rc = siridb_points_raw_pack(
+ (siridb_points_t * ) plist->data[i],
+ query->packer);
+ }
+
+ return -(rc || qp_add_type(query->packer, QP_ARRAY_CLOSE));
+}
+
+static void on_select_unpack_points(
+ qp_unpacker_t * unpacker,
+ query_select_t * q_select,
+ qp_obj_t * qp_name,
+ qp_obj_t * qp_tp,
+ qp_obj_t * qp_len,
+ qp_obj_t * qp_points,
+ uint32_t select_points_limit)
+{
+ siridb_points_t * points;
+
+ while ( q_select->n <= select_points_limit &&
+ qp_is_raw(qp_next(unpacker, qp_name)) &&
+ qp_is_raw_term(qp_name) &&
+ qp_is_array(qp_next(unpacker, NULL)) &&
+ qp_is_int(qp_next(unpacker, qp_tp)) &&
+ qp_is_int(qp_next(unpacker, qp_len)) &&
+ qp_is_raw(qp_next(unpacker, qp_points)))
+ {
+ points = siridb_points_new(qp_len->via.int64, qp_tp->via.int64);
+ if (points != NULL)
+ {
+ if (points->tp == TP_STRING)
+ {
+ if (qp_len->via.int64 < POINTS_ZIP_THRESHOLD)
+ {
+ siridb_points_unzip_string_raw(
+ points,
+ qp_points->via.raw,
+ qp_len->via.int64);
+ }
+ else
+ {
+ siridb_points_unzip_string(
+ points,
+ qp_points->via.raw,
+ qp_len->via.int64,
+ NULL, NULL, 0);
+ }
+ }
+ else
+ {
+ points->len = qp_len->via.int64;
+ memcpy(points->data, qp_points->via.raw, qp_points->len);
+ }
+
+ if (ct_add(q_select->result, (char *) qp_name->via.raw, points))
+ {
+ siridb_points_free(points);
+ }
+ else
+ {
+ q_select->n += points->len;
+ }
+ }
+
+ qp_next(unpacker, NULL); /* QP_ARRAY_CLOSE */
+ }
+}
+
+static void on_select_unpack_merged_points(
+ qp_unpacker_t * unpacker,
+ query_select_t * q_select,
+ qp_obj_t * qp_name,
+ qp_obj_t * qp_tp,
+ qp_obj_t * qp_len,
+ qp_obj_t * qp_points,
+ uint32_t select_points_limit)
+{
+ siridb_points_t * points;
+
+ while ( qp_is_raw(qp_next(unpacker, qp_name)) &&
+ qp_is_raw_term(qp_name) &&
+ qp_is_array(qp_next(unpacker, NULL)))
+ {
+ vec_t ** plist = (vec_t **) ct_getaddr(
+ q_select->result,
+ (const char *) qp_name->via.raw);
+
+ while ( q_select->n <= select_points_limit &&
+ qp_is_array(qp_next(unpacker, NULL)) &&
+ qp_is_int(qp_next(unpacker, qp_tp)) &&
+ qp_is_int(qp_next(unpacker, qp_len)) &&
+ qp_is_raw(qp_next(unpacker, qp_points)))
+ {
+
+ points = siridb_points_new(qp_len->via.int64, qp_tp->via.int64);
+
+ if (points != NULL)
+ {
+ if (points->tp == TP_STRING)
+ {
+ if (qp_len->via.int64 < POINTS_ZIP_THRESHOLD)
+ {
+ siridb_points_unzip_string_raw(
+ points,
+ qp_points->via.raw,
+ qp_len->via.int64);
+ }
+ else
+ {
+ siridb_points_unzip_string(
+ points,
+ qp_points->via.raw,
+ qp_len->via.int64,
+ NULL, NULL, 0);
+ }
+ }
+ else
+ {
+ points->len = qp_len->via.int64;
+ memcpy(points->data, qp_points->via.raw, qp_points->len);
+ }
+
+ if (vec_append_safe(plist, points))
+ {
+ siridb_points_free(points);
+ }
+ else
+ {
+ q_select->n += points->len;
+ }
+ }
+
+ qp_next(unpacker, NULL); /* QP_ARRAY_CLOSE */
+ }
+ }
+}
+
+static int values_list_groups(siridb_group_t * group, uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ cexpr_t * where_expr = ((query_list_t *) query->data)->where_expr;
+ cexpr_cb_t cb = (cexpr_cb_t) siridb_group_cexpr_cb;
+ vec_t * props = ((query_list_t *) query->data)->props;
+
+ if (where_expr == NULL || cexpr_run(where_expr, cb, group))
+ {
+ size_t i;
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ for (i = 0; i < props->len; i++)
+ {
+ siridb_group_prop(
+ group,
+ query->packer,
+ *((uint32_t *) props->data[i]));
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ return 1;
+ }
+
+ return 0;
+}
+
+static int values_count_groups(siridb_group_t * group, uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ return cexpr_run(
+ ((query_list_t *) query->data)->where_expr,
+ (cexpr_cb_t) siridb_group_cexpr_cb,
+ group);
+}
+
+static void finish_list_groups(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_list_t * q_list = (query_list_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ if (q_list->props == NULL)
+ {
+ q_list->props = vec_new(1);
+ if (q_list->props == NULL)
+ {
+ MEM_ERR_RET
+ }
+ vec_append(q_list->props, &GID_K_NAME);
+ qp_add_raw(query->packer, (const unsigned char *) "name", 4);
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ qp_add_raw(query->packer, (const unsigned char *) "groups", 6);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ ct_valuesn(
+ siridb->groups->groups,
+ &q_list->limit,
+ (ct_val_cb) values_list_groups,
+ handle);
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void finish_count_groups(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_count_t * q_count = (query_count_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ /* Note: ct_values(..values_count_groups..) can only result in a positive
+ * value.
+ */
+ size_t n = (q_count->where_expr == NULL) ?
+ siridb->groups->groups->len :
+ (size_t) ct_values(
+ siridb->groups->groups,
+ (ct_val_cb) values_count_groups,
+ handle);
+
+ qp_add_raw(query->packer, (const unsigned char *) "groups", 6);
+
+ qp_add_int64(query->packer, n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static int values_list_tags(siridb_tag_t * tag, uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ cexpr_t * where_expr = ((query_list_t *) query->data)->where_expr;
+ cexpr_cb_t cb = (cexpr_cb_t) siridb_tag_cexpr_cb;
+ vec_t * props = ((query_list_t *) query->data)->props;
+
+ if (where_expr == NULL || cexpr_run(where_expr, cb, tag))
+ {
+ size_t i;
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ for (i = 0; i < props->len; i++)
+ {
+ siridb_tag_prop(
+ tag,
+ query->packer,
+ *((uint32_t *) props->data[i]));
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ return 1;
+ }
+
+ return 0;
+}
+
+static int values_count_tags(siridb_tag_t * tag, uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+
+ return cexpr_run(
+ ((query_list_t *) query->data)->where_expr,
+ (cexpr_cb_t) siridb_tag_cexpr_cb,
+ tag);
+}
+
+static void finish_list_tags(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_list_t * q_list = (query_list_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ if (q_list->props == NULL)
+ {
+ q_list->props = vec_new(1);
+ if (q_list->props == NULL)
+ {
+ MEM_ERR_RET
+ }
+ vec_append(q_list->props, &GID_K_NAME);
+ qp_add_raw(query->packer, (const unsigned char *) "name", 4);
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ qp_add_raw(query->packer, (const unsigned char *) "tags", 4);
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ ct_valuesn(
+ siridb->tags->tags,
+ &q_list->limit,
+ (ct_val_cb) values_list_tags,
+ handle);
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
+
+static void finish_count_tags(uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_count_t * q_count = (query_count_t *) query->data;
+ siridb_t * siridb = query->client->siridb;
+
+ /* Note: ct_values(..values_count_tags..) can only result in a positive
+ * value.
+ */
+ size_t n = (q_count->where_expr == NULL) ?
+ siridb->tags->tags->len :
+ (size_t) ct_values(
+ siridb->tags->tags,
+ (ct_val_cb) values_count_tags,
+ handle);
+
+ qp_add_raw(query->packer, (const unsigned char *) "tags", 4);
+
+ qp_add_int64(query->packer, n);
+
+ SIRIPARSER_ASYNC_NEXT_NODE
+}
--- /dev/null
+/*
+ * lookup.c - Find and assign to which pool series belong.
+ */
+#include <siri/db/lookup.h>
+#include <siri/err.h>
+#include <stdlib.h>
+#include <string.h>
+
+/*
+ * Returns a pool id based on a terminated string.
+ */
+uint16_t siridb_lookup_sn(siridb_lookup_t * lookup, const char * sn)
+{
+ uint32_t n = 0;
+ for (; *sn; sn++)
+ {
+ n += *sn;
+ }
+ return (*lookup)[n % SIRIDB_LOOKUP_SZ];
+}
+
+/*
+ * Returns a pool id based on a raw string.
+ */
+uint16_t siridb_lookup_sn_raw(
+ siridb_lookup_t * lookup,
+ const char * sn,
+ size_t len)
+{
+ uint32_t n = 0;
+ while (len--)
+ {
+ n += sn[len];
+ }
+ return (*lookup)[n % SIRIDB_LOOKUP_SZ];
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ *
+ * (Algorithm to create pools lookup array.)
+ */
+siridb_lookup_t * siridb_lookup_new(uint_fast16_t num_pools)
+{
+ siridb_lookup_t * lookup = calloc(1, sizeof(siridb_lookup_t));
+
+ if (lookup == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ uint_fast16_t n, i, m;
+ uint_fast16_t counters[num_pools - 1];
+
+ for (n = 1, m = 2; n < num_pools; n++, m++)
+ {
+ for (i = 0; i < n; i++)
+ {
+ counters[i] = i;
+ }
+
+ for (i = 0; i < SIRIDB_LOOKUP_SZ; i++)
+ {
+ if (++counters[ (*lookup)[i] ] % m == 0)
+ {
+ (*lookup)[i] = n;
+ }
+ }
+ }
+ }
+ return lookup;
+}
+
+/*
+ * Destroy lookup.
+ */
+void siridb_lookup_free(siridb_lookup_t * lookup)
+{
+ free(lookup);
+}
+
+
--- /dev/null
+/*
+ * median.c - Calculate median, median high and median low.
+ */
+#include <siri/db/median.h>
+#include <siri/db/points.h>
+#include <stdbool.h>
+#include <assert.h>
+#include <logger/logger.h>
+
+static int64_t find_n_int64(
+ int64_t * seq,
+ uint64_t size,
+ int64_t * other,
+ uint64_t n);
+
+static double find_n_double(
+ double * seq,
+ uint64_t size,
+ double * other,
+ uint64_t n);
+
+static double find_median_real_int64(
+ int64_t * seq,
+ uint64_t size,
+ int64_t * other,
+ uint64_t n,
+ int64_t a,
+ int64_t b,
+ bool found_a,
+ bool found_b);
+
+static double find_median_real_double(
+ double * seq,
+ uint64_t size,
+ double * other,
+ uint64_t n,
+ double a,
+ double b,
+ bool found_a,
+ bool found_b);
+
+int siridb_median_find_n(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ uint64_t n)
+{
+ assert (points->len >= 2);
+ int rc = 0;
+ uint64_t i, npivot, size_l, size_r;
+
+ size_l = size_r = 0;
+ npivot = points->len / 2;
+
+ if (points->tp == TP_INT)
+ {
+ int64_t pivot, v;
+
+ int64_t * arr_l = malloc(sizeof(int64_t) * 2 * (points->len - 1));
+
+ if (arr_l == NULL)
+ {
+ log_critical("Memory allocation error occurred.");
+ rc = -1;
+ }
+ else
+ {
+ int64_t * arr_r = arr_l + points->len - 1;
+ int64_t * equal = arr_l;
+ uint64_t * equal_size = &size_l;
+
+ pivot = points->data[npivot].val.int64;
+
+ for (i = 0; i < points->len; i++)
+ {
+ if (i == npivot)
+ {
+ continue;
+ }
+ v = points->data[i].val.int64;
+ if (v == pivot)
+ {
+ equal[(*equal_size)++] = v;
+ if (equal == arr_l)
+ {
+ equal = arr_r;
+ equal_size = &size_r;
+ }
+ else
+ {
+ equal = arr_l;
+ equal_size = &size_l;
+ }
+ }
+ else if (v > pivot)
+ {
+ arr_r[size_r++] = v;
+ }
+ else
+ {
+ arr_l[size_l++] = v;
+ }
+ }
+ point->val.int64 = (size_l == n) ? pivot : ((size_l > n) ?
+ find_n_int64(arr_l, size_l, arr_r, n) :
+ find_n_int64(arr_r, size_r, arr_l, n - size_l - 1));
+ }
+ free(arr_l);
+ }
+ else
+ {
+ double pivot, v;
+
+ double * arr_l = malloc(sizeof(double) * 2 * (points->len - 1));
+
+ if (arr_l == NULL)
+ {
+ log_critical("Memory allocation error occurred.");
+ rc = -1;
+ }
+ else
+ {
+ double * arr_r = arr_l + points->len - 1;
+ double * equal = arr_l;
+ uint64_t * equal_size = &size_l;
+
+ pivot = points->data[npivot].val.real;
+ for (i = 0; i < points->len; i++)
+ {
+ if (i == npivot)
+ {
+ continue;
+ }
+ v = points->data[i].val.real;
+ if (v == pivot)
+ {
+ equal[(*equal_size)++] = v;
+ if (equal == arr_l)
+ {
+ equal = arr_r;
+ equal_size = &size_r;
+ }
+ else
+ {
+ equal = arr_l;
+ equal_size = &size_l;
+ }
+ }
+ else if (v > pivot)
+ {
+ arr_r[size_r++] = v;
+ }
+ else
+ {
+ arr_l[size_l++] = v;
+ }
+ }
+
+ point->val.real = (size_l == n) ? pivot : (size_l > n) ?
+ find_n_double(arr_l, size_l, arr_r, n) :
+ find_n_double(arr_r, size_r, arr_l, n - size_l - 1);
+ }
+ free(arr_l);
+ }
+ return rc;
+}
+
+int siridb_median_real(
+ siridb_point_t * point,
+ siridb_points_t * points,
+ double percentage)
+{
+ assert (points->len >= 2);
+ int rc = 0;
+ uint64_t i, npivot, size_l, size_r, n;
+ bool found_a, found_b;
+
+ n = points->len * percentage;
+
+ size_l = size_r = 0;
+ found_a = found_b = false;
+ npivot = points->len / 2;
+
+ if (points->tp == TP_INT)
+ {
+ int64_t pivot, v, a, b;
+
+ int64_t * arr_l = malloc(sizeof(int64_t) * 2 * (points->len - 1));
+
+ if (arr_l == NULL)
+ {
+ log_critical("Memory allocation error occurred.");
+ rc = -1;
+ }
+ else
+ {
+ int64_t * arr_r = arr_l + points->len - 1;
+ int64_t * equal = arr_l;
+ uint64_t * equal_size = &size_l;
+
+ a = b = 0;
+ pivot = points->data[npivot].val.int64;
+ for (i = 0; i < points->len; i++)
+ {
+ if (i == npivot)
+ {
+ continue;
+ }
+ v = points->data[i].val.int64;
+ if (v == pivot)
+ {
+ equal[(*equal_size)++] = v;
+ if (equal == arr_l)
+ {
+ equal = arr_r;
+ equal_size = &size_r;
+ }
+ else
+ {
+ equal = arr_l;
+ equal_size = &size_l;
+ }
+ }
+ else if (v > pivot)
+ {
+ arr_r[size_r++] = v;
+ }
+ else
+ {
+ arr_l[size_l++] = v;
+ }
+ }
+
+ if (size_l == n - 1)
+ {
+ a = pivot;
+ found_a = true;
+ }
+ else if (size_l == n)
+ {
+ b = pivot;
+ found_b = true;
+ }
+
+ point->val.real = ((!found_b && size_l > n) || size_l > n - 1) ?
+ find_median_real_int64(
+ arr_l, size_l, arr_r, n, a, b, found_a, found_b) :
+ find_median_real_int64(
+ arr_r,
+ size_r,
+ arr_l,
+ n - size_l - 1,
+ a,
+ b,
+ found_a,
+ found_b);
+ }
+ free(arr_l);
+ }
+ else
+ {
+ double pivot, v, a, b;
+
+ double * arr_l = malloc(sizeof(double) * 2 * (points->len - 1));
+
+ if (arr_l == NULL)
+ {
+ log_critical("Memory allocation error occurred.");
+ rc = -1;
+ }
+ else
+ {
+ double * arr_r = arr_l + points->len - 1;
+ double * equal = arr_l;
+ uint64_t * equal_size = &size_l;
+
+ a = b = 0.0;
+ pivot = points->data[npivot].val.real;
+ for (i = 0; i < points->len; i++)
+ {
+ if (i == npivot)
+ {
+ continue;
+ }
+ v = points->data[i].val.real;
+ if (v == pivot)
+ {
+ equal[(*equal_size)++] = v;
+ if (equal == arr_l)
+ {
+ equal = arr_r;
+ equal_size = &size_r;
+ }
+ else
+ {
+ equal = arr_l;
+ equal_size = &size_l;
+ }
+ }
+ else if (v > pivot)
+ {
+ arr_r[size_r++] = v;
+ }
+ else
+ {
+ arr_l[size_l++] = v;
+ }
+ }
+ if (size_l == n - 1)
+ {
+ a = pivot;
+ found_a = true;
+ }
+ else if (size_l == n)
+ {
+ b = pivot;
+ found_b = true;
+ }
+ point->val.real = ((!found_b && size_l > n) || size_l > n - 1) ?
+ find_median_real_double(
+ arr_l, size_l, arr_r, n, a, b, found_a, found_b) :
+ find_median_real_double(
+ arr_r,
+ size_r,
+ arr_l,
+ n - size_l - 1,
+ a,
+ b,
+ found_a,
+ found_b);
+ }
+ free(arr_l);
+ }
+ return rc;
+}
+
+static double find_median_real_int64(
+ int64_t * seq,
+ uint64_t size,
+ int64_t * other,
+ uint64_t n,
+ int64_t a,
+ int64_t b,
+ bool found_a,
+ bool found_b)
+{
+ uint64_t i, size_l, size_r;
+ size_l = size_r = 0;
+ int64_t pivot, v;
+ int64_t * equal = seq;
+ uint64_t * equal_size = &size_l;
+ uint64_t npivot = size / 2;
+
+ pivot = seq[npivot];
+
+ for (i = 0; i < size; i++)
+ {
+ if (i == npivot)
+ {
+ continue;
+ }
+ v = seq[i];
+ if (v == pivot)
+ {
+ equal[(*equal_size)++] = v;
+ if (equal == seq)
+ {
+ equal = other;
+ equal_size = &size_r;
+ }
+ else
+ {
+ equal = seq;
+ equal_size = &size_l;
+ }
+ }
+ else if (v > pivot)
+ {
+ other[size_r++] = v;
+ }
+ else
+ {
+ seq[size_l++] = v;
+ }
+ }
+
+ if (size_l == n - 1)
+ {
+ a = pivot;
+ found_a = true;
+ }
+ else if (size_l == n)
+ {
+ b = pivot;
+ found_b = true;
+ }
+
+ if (found_a && found_b)
+ {
+ return ((double) a + (double) b) / 2.0f;
+ }
+
+ if ((!found_b && size_l > n) || size_l > n - 1)
+ {
+ return find_median_real_int64(
+ seq, size_l, other, n, a, b, found_a, found_b);
+ }
+
+ return find_median_real_int64(
+ other, size_r, seq, n - size_l - 1, a, b, found_a, found_b);
+
+}
+
+static double find_median_real_double(
+ double * seq,
+ uint64_t size,
+ double * other,
+ uint64_t n,
+ double a,
+ double b,
+ bool found_a,
+ bool found_b)
+{
+ uint64_t i, size_l, size_r;
+ size_l = size_r = 0;
+ double pivot, v;
+ double * equal = seq;
+ uint64_t * equal_size = &size_l;
+ uint64_t npivot = size / 2;
+
+ pivot = seq[npivot];
+
+ for (i = 0; i < size; i++)
+ {
+ if (i == npivot)
+ {
+ continue;
+ }
+ v = seq[i];
+ if (v == pivot)
+ {
+ equal[(*equal_size)++] = v;
+ if (equal == seq)
+ {
+ equal = other;
+ equal_size = &size_r;
+ }
+ else
+ {
+ equal = seq;
+ equal_size = &size_l;
+ }
+ }
+ else if (v > pivot)
+ {
+ other[size_r++] = v;
+ }
+ else
+ {
+ seq[size_l++] = v;
+ }
+ }
+ if (size_l == n - 1)
+ {
+ a = pivot;
+ found_a = true;
+ }
+ else if (size_l == n)
+ {
+ b = pivot;
+ found_b = true;
+ }
+
+ if (found_a && found_b)
+ {
+ return (a + b) / 2.0;
+ }
+
+ if ((!found_b && size_l > n) || size_l > n - 1)
+ {
+ return find_median_real_double(
+ seq, size_l, other, n, a, b, found_a, found_b);
+ }
+
+ return find_median_real_double(
+ other, size_r, seq, n - size_l - 1, a, b, found_a, found_b);
+
+}
+
+static int64_t find_n_int64(
+ int64_t * seq,
+ uint64_t size,
+ int64_t * other,
+ uint64_t n)
+{
+ uint64_t i, size_l, size_r;
+ size_l = size_r = 0;
+ int64_t pivot, v;
+ int64_t * equal = seq;
+ uint64_t * equal_size = &size_l;
+ uint64_t npivot = size / 2;
+
+ pivot = seq[npivot];
+
+ for (i = 0; i < size; i++)
+ {
+ if (i == npivot)
+ {
+ continue;
+ }
+ v = seq[i];
+ if (v == pivot)
+ {
+ equal[(*equal_size)++] = v;
+ if (equal == seq)
+ {
+ equal = other;
+ equal_size = &size_r;
+ }
+ else
+ {
+ equal = seq;
+ equal_size = &size_l;
+ }
+ }
+ else if (v > pivot)
+ {
+ other[size_r++] = v;
+ }
+ else
+ {
+ seq[size_l++] = v;
+ }
+ }
+
+ if (size_l == n)
+ {
+ return pivot;
+ }
+
+ if (size_l > n)
+ {
+ return find_n_int64(seq, size_l, other, n);
+ }
+
+ return find_n_int64(other, size_r, seq, n - size_l - 1);
+}
+
+static double find_n_double(
+ double * seq,
+ uint64_t size,
+ double * other,
+ uint64_t n)
+{
+ uint64_t i, size_l, size_r;
+ size_l = size_r = 0;
+ double pivot, v;
+ double * equal = seq;
+ uint64_t * equal_size = &size_l;
+ uint64_t npivot = size / 2;
+
+ pivot = seq[npivot];
+
+ for (i = 0; i < size; i++)
+ {
+ if (i == npivot)
+ {
+ continue;
+ }
+ v = seq[i];
+ if (v == pivot)
+ {
+ equal[(*equal_size)++] = v;
+ if (equal == seq)
+ {
+ equal = other;
+ equal_size = &size_r;
+ }
+ else
+ {
+ equal = seq;
+ equal_size = &size_l;
+ }
+ }
+ else if (v > pivot)
+ {
+ other[size_r++] = v;
+ }
+ else
+ {
+ seq[size_l++] = v;
+ }
+ }
+ if (size_l == n)
+ {
+ return pivot;
+ }
+
+ if (size_l > n)
+ {
+ return find_n_double(seq, size_l, other, n);
+ }
+
+ return find_n_double(other, size_r, seq, n - size_l - 1);
+}
--- /dev/null
+/*
+ * misc.c - Miscellaneous functions used by SiriDB.
+ */
+#include <siri/db/misc.h>
+#include <stdlib.h>
+#include <siri/err.h>
+
+qp_unpacker_t * siridb_misc_open_schema_file(uint8_t schema, const char * fn)
+{
+ qp_unpacker_t * unpacker = qp_unpacker_ff(fn);
+ if (unpacker != NULL)
+ {
+ qp_obj_t qp_schema;
+ if ( !qp_is_array(qp_next(unpacker, NULL)) ||
+ qp_next(unpacker, &qp_schema) != QP_INT64 ||
+ qp_schema.via.int64 != schema)
+ {
+ log_critical("Invalid schema detected in '%s'", fn);
+ qp_unpacker_ff_free(unpacker);
+ unpacker = NULL;
+ }
+ }
+
+ return unpacker;
+}
--- /dev/null
+/*
+ * nodes.c - Contains logic for cleri nodes which we need to parse.
+ */
+#include <logger/logger.h>
+#include <siri/db/nodes.h>
+#include <stdlib.h>
+#include <stddef.h>
+
+void siridb_nodes_next(siridb_nodes_t ** nodes)
+{
+ siridb_nodes_t * next = (*nodes)->next;
+ free(*nodes);
+ *nodes = next;
+}
+
+void siridb_nodes_free(siridb_nodes_t * nodes)
+{
+ /* the node self will be closed when freeing the parse result */
+ siridb_nodes_t * next;
+
+ while (nodes != NULL)
+ {
+ next = nodes->next;
+ free(nodes);
+ nodes = next;
+ }
+}
--- /dev/null
+/*
+ * pcache.c - Points structure with notion of its size.
+ */
+#include <assert.h>
+#include <siri/db/pcache.h>
+#include <siri/err.h>
+#include <stddef.h>
+
+#define PCACHE_DEFAULT_SIZE 64
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ *
+ * A pcache object can be parsed to all points functions.
+ */
+siridb_pcache_t * siridb_pcache_new(points_tp tp)
+{
+ siridb_pcache_t * pcache = malloc(sizeof(siridb_pcache_t));
+ if (pcache == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ pcache->size = PCACHE_DEFAULT_SIZE;
+ pcache->len = 0;
+ pcache->tp = tp;
+ pcache->data = malloc(sizeof(siridb_point_t) * PCACHE_DEFAULT_SIZE);
+ if (pcache->data == NULL)
+ {
+ ERR_ALLOC
+ free(pcache);
+ pcache = NULL;
+ }
+ }
+ return pcache;
+}
+
+/*
+ * Add a point to points. (points are sorted by timestamp so the new point
+ * will be inserted at the correct position.
+ *
+ * Returns 0 if successful or -1 and a signal is raised in case of an error.
+ */
+int siridb_pcache_add_point(
+ siridb_pcache_t * pcache,
+ uint64_t * ts,
+ qp_via_t * val)
+{
+ if (pcache->len == pcache->size)
+ {
+ siridb_point_t * tmp;
+ pcache->size *= 2;
+ tmp = realloc(pcache->data, sizeof(siridb_point_t) * pcache->size);
+ if (tmp == NULL)
+ {
+ log_error(
+ "Cannot re-allocate memory for %lu points",
+ pcache->size);
+
+ ERR_ALLOC
+ /* restore original size */
+ pcache->size /= 2;
+ return -1;
+ }
+ pcache->data = tmp;
+ }
+
+ siridb_points_add_point((siridb_points_t *) pcache, ts, val);
+
+ return 0;
+}
--- /dev/null
+/*
+ * points.c - Array object for points.
+ */
+#include <siri/db/points.h>
+#include <logger/logger.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <siri/err.h>
+#include <unistd.h>
+#include <string.h>
+#include <xstr/xstr.h>
+
+#define MAX_ITERATE_MERGE_COUNT 1000
+#define POINTS_MAX_QSORT 250000
+#define RAW_VALUES_THRESHOLD 7
+#define DICT_SZ 0x3fff
+#define TOLERANCE_INTERVAL_DETECT 10
+
+static unsigned char * POINTS_zip_raw(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size);
+static void POINTS_unzip_raw(
+ siridb_points_t * points,
+ unsigned char * bits,
+ uint16_t len,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap);
+static void POINTS_sort_while_merge(vec_t * plist, siridb_points_t * points);
+static void POINTS_merge_and_sort(vec_t * plist, siridb_points_t * points);
+static void POINTS_simple_sort(siridb_points_t * points);
+static inline int POINTS_compare(const void * a, const void * b);
+static void POINTS_highest_and_merge(vec_t * plist, siridb_points_t * points);
+static size_t POINTS_strlen_check_ascii(const char * str, uint8_t * is_ascii);
+static void POINTS_output_literal(
+ size_t len,
+ uint8_t * pt,
+ uint8_t ** out,
+ uint8_t is_ascii);
+static void POINTS_output_match(
+ size_t offset,
+ size_t len,
+ uint8_t ** out,
+ uint8_t is_ascii);
+static void POINTS_zip_str(
+ uint8_t ** out,
+ uint8_t ** pt,
+ uint8_t * src,
+ uint8_t * s,
+ size_t n,
+ uint8_t is_ascii);
+static int POINTS_set_cinfo_size(uint16_t * cinfo, size_t * size);
+inline static uint16_t POINTS_hash(uint32_t h);
+static void POINTS_destroy(siridb_points_t * points);
+
+static uint8_t * dictionary[DICT_SZ + 1];
+
+void siridb_points_init(void)
+{
+ memset(dictionary, 0, sizeof(dictionary));
+}
+
+/*
+ * Returns NULL in case an error has occurred.
+ */
+siridb_points_t * siridb_points_new(size_t size, points_tp tp)
+{
+ siridb_points_t * points = malloc(sizeof(siridb_points_t));
+ if (points == NULL)
+ {
+ return NULL;
+ }
+
+ points->len = 0;
+ points->tp = tp;
+ points->data = malloc(sizeof(siridb_point_t) * size);
+ if (points->data == NULL)
+ {
+ free(points);
+ return NULL;
+ }
+
+ return points;
+}
+
+/*
+ * Resize points to a new size. Returns 0 when successful or -1 if failed.
+ */
+int siridb_points_resize(siridb_points_t * points, size_t n)
+{
+ assert( points->len <= n );
+ siridb_point_t * tmp = realloc(points->data, sizeof(siridb_point_t) * n);
+ if (tmp == NULL && n)
+ {
+ return -1;
+ }
+ points->data = tmp;
+ return 0;
+}
+
+/*
+ * Returns a copy of points or NULL in case of an error. NULL is also returned
+ * if points is NULL.
+ */
+siridb_points_t * siridb_points_copy(siridb_points_t * points)
+{
+ if (points == NULL)
+ {
+ return NULL;
+ }
+ siridb_points_t * cpoints = malloc(sizeof(siridb_points_t));
+ if (cpoints != NULL)
+ {
+ size_t sz = sizeof(siridb_point_t) * points->len;
+ cpoints->len = points->len;
+ cpoints->tp = points->tp;
+ cpoints->data = malloc(sz);
+ if (cpoints->data == NULL)
+ {
+ free(cpoints);
+ cpoints = NULL;
+ }
+ else
+ {
+ memcpy(cpoints->data, points->data, sz);
+ }
+ if (points->tp == TP_STRING)
+ {
+ size_t i;
+ for (i = 0; i < points->len; ++i)
+ {
+ (cpoints->data + i)->val.str =
+ strdup((points->data + i)->val.str);
+ }
+ }
+ }
+ return cpoints;
+}
+
+/*
+ * Destroy points. (parsing NULL is NOT allowed)
+ */
+void siridb_points_free(siridb_points_t * points)
+{
+ if (points->tp == TP_STRING)
+ {
+ size_t i;
+ for (i = 0; i < points->len; ++i)
+ {
+ free((points->data + i)->val.str);
+ }
+ }
+ free(points->data);
+ free(points);
+}
+
+/*
+ * Add a point to points. (points are sorted by timestamp so the new point
+ * will be inserted at the correct position.
+ *
+ * Warning:
+ * this functions assumes points to be large enough to hold the new
+ * point and is therefore not safe.
+ */
+void siridb_points_add_point(
+ siridb_points_t *__restrict points,
+ uint64_t * ts,
+ qp_via_t * val)
+{
+ siridb_point_t * first = points->data;
+ siridb_point_t * point = points->data + points->len;
+ siridb_point_t * prev = point - 1;
+
+ while (prev >= first && prev->ts > *ts)
+ {
+ *point = *prev;
+ --point;
+ --prev;
+ }
+
+ ++points->len;
+ point->ts = *ts;
+ point->val = *val;
+}
+
+/*
+ * Returns siri_err and raises a SIGNAL in case an error has occurred.
+ */
+int siridb_points_pack(siridb_points_t * points, qp_packer_t * packer)
+{
+ qp_add_type(packer, QP_ARRAY_OPEN);
+ if (points->len)
+ {
+ size_t i;
+ siridb_point_t * point = points->data;
+ switch (points->tp)
+ {
+ case TP_INT:
+ for (i = 0; i < points->len; i++, point++)
+ {
+ qp_add_type(packer, QP_ARRAY2);
+ qp_add_int64(packer, (int64_t) point->ts);
+ qp_add_int64(packer, point->val.int64);
+ }
+ break;
+ case TP_DOUBLE:
+ for (i = 0; i < points->len; i++, point++)
+ {
+ qp_add_type(packer, QP_ARRAY2);
+ qp_add_int64(packer, (int64_t) point->ts);
+ qp_add_double(packer, point->val.real);
+ }
+ break;
+ case TP_STRING:
+ for (i = 0; i < points->len; i++, point++)
+ {
+ qp_add_type(packer, QP_ARRAY2);
+ qp_add_int64(packer, (int64_t) point->ts);
+ qp_add_string(packer, point->val.str);
+ }
+ break;
+ }
+ }
+ qp_add_type(packer, QP_ARRAY_CLOSE);
+
+ return siri_err;
+}
+
+void siridb_points_ts_correction(siridb_points_t * points, double factor)
+{
+ siridb_point_t * point = points->data;
+ size_t i;
+ for (i = 0; i < points->len; i++, point++)
+ {
+ point->ts *= factor;
+ }
+}
+
+/*
+ * Returns 0 if successful or -1 and a SIGNAL is raised in case of an error.
+ */
+int siridb_points_raw_pack(siridb_points_t * points, qp_packer_t * packer)
+{
+ int rc;
+ size_t size;
+ unsigned char * data;
+ if (points->tp == TP_STRING)
+ {
+ uint16_t cinfo;
+ data = siridb_points_zip_string(points, 0, points->len, &cinfo, &size);
+ }
+ else
+ {
+ data = (unsigned char *) points->data;
+ size = points->len * sizeof(siridb_point_t);
+ }
+
+ rc = -(qp_add_type(packer, QP_ARRAY_OPEN) ||
+ qp_add_int64(packer, (int64_t) points->tp) ||
+ qp_add_int64(packer, (int64_t) points->len) ||
+ qp_add_raw(packer, data, size) ||
+ qp_add_type(packer, QP_ARRAY_CLOSE));
+
+ if (points->tp == TP_STRING)
+ {
+ free(data);
+ }
+
+ return rc;
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ * (err_msg is set when an error has occurred)
+ * Use this function only when having at least two 'series' in the list.
+ */
+siridb_points_t * siridb_points_merge(vec_t * plist, char * err_msg)
+{
+ assert (plist->len >= 2);
+ siridb_points_t * points;
+ siridb_points_t * tpts = NULL;
+ size_t n = 0;
+ size_t i;
+ uint8_t int2double = 0;
+ for (i = 0; i < plist->len; )
+ {
+ points = (siridb_points_t *) plist->data[i];
+
+ if (!points->len)
+ {
+ if (!--plist->len)
+ {
+ /* all series are empty, return the last one */
+ return plist->data[i];
+ }
+ /* cleanup empty points */
+ POINTS_destroy(plist->data[i]);
+
+ /* shrink plist length and fill gap */
+ plist->data[i] = plist->data[plist->len];
+
+ continue;
+ }
+
+ n += points->len;
+
+ if (tpts != NULL && tpts->tp != points->tp)
+ {
+ if (tpts->tp == TP_STRING || points->tp == TP_STRING)
+ {
+ sprintf(err_msg, "Cannot merge string and number series.");
+ return NULL;
+ }
+ int2double = 1;
+ }
+
+ /*
+ * Decrement the points->len so we can use this not truly as length
+ * property but as position of the last element.
+ */
+ points->len--;
+ tpts = points;
+ i++;
+ }
+
+ /* we have at least one series left */
+ assert (plist->len >= 1);
+
+ if (plist->len == 1)
+ {
+ /*
+ * Return the only left points since there is nothing to merge as set
+ * list length to 0. We do have to restore the points length since
+ * this is decremented by one.
+ */
+ points = (siridb_points_t *) vec_pop(plist);
+ points->len++;
+ return points;
+ }
+
+ points = siridb_points_new(n, (int2double) ? TP_DOUBLE : tpts->tp);
+
+ if (points == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ }
+ else
+ {
+ usleep(1000);
+
+ /*
+ * When both series from type double and type integer are merged
+ * we need to promote the integer series to double.
+ */
+ if (int2double)
+ {
+ size_t j;
+ for (i = 0; i < plist->len; i++)
+ {
+ tpts = (siridb_points_t *) plist->data[i];
+ if (tpts->tp == TP_INT)
+ {
+ for (j = 0; j <= tpts->len; j++)
+ {
+ tpts->data[j].val.real =
+ (double) tpts->data[j].val.int64;
+ }
+ usleep(100);
+ }
+ }
+ }
+
+ points->len = n;
+
+ /*
+ * Select a merge method depending on the ratio series and points.
+ */
+ if (plist->len < 4)
+ {
+ POINTS_sort_while_merge(plist, points);
+ }
+ else if (n == plist->len || n < POINTS_MAX_QSORT)
+ {
+ POINTS_merge_and_sort(plist, points);
+ }
+ else
+ {
+ POINTS_highest_and_merge(plist, points);
+ }
+ }
+ return points;
+}
+
+/*
+ * Return NULL in case an error has occurred. This function destroys the
+ * original points.
+ */
+unsigned char * siridb_points_zip_int(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size)
+{
+ if (end - start < POINTS_ZIP_THRESHOLD)
+ {
+ return POINTS_zip_raw(points, start, end, cinfo, size);
+ }
+ const uint64_t store_raw_mask = UINT64_C(0x8000000000000000);
+ uint64_t tdiff = 0;
+ uint64_t vdiff = 0;
+ unsigned char * bits, *pt;
+ uint64_t mask, val;
+ size_t i = end - 2;
+ siridb_point_t * point = points->data + i;
+ uint64_t ts = (point + 1)->ts - point->ts;
+ uint8_t vcount = 0;
+ uint8_t tcount = 0;
+ uint8_t shift = 0;
+ int64_t a, b;
+
+ a = (point + 1)->val.int64;
+ b = point->val.int64;
+
+ vdiff |= (uint64_t) (a > b) ? a - b : b - a;
+ while (i-- > start)
+ {
+ point--;
+ tdiff |= ts ^ ((point + 1)->ts - point->ts);
+ a = b;
+ b = point->val.int64;
+ vdiff |= (uint64_t) (a > b) ? a - b : b - a;
+ }
+
+ vdiff <<= !(vdiff & store_raw_mask);
+
+ for (i = 1, mask = 0xff; i <= 8; mask <<= 8, i++)
+ {
+ if (vdiff & mask)
+ {
+ vcount = i;
+ }
+ if (tdiff & mask)
+ {
+ tcount = i;
+ }
+ else if (ts & mask)
+ {
+ shift = i;
+ }
+ }
+
+ *cinfo = 0xff00 >> vcount;
+ shift = (tcount > shift) ? tcount : shift;
+ *cinfo <<= 8;
+ *cinfo |= tcount | (shift << 4);
+ *size = 16 + shift - tcount + (tcount + vcount) * (end - start - 1);
+ bits = malloc(*size);
+ if (bits == NULL)
+ {
+ return NULL;
+ }
+ pt = bits;
+
+ memcpy(pt, &point->ts, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ memcpy(pt, &point->val.uint64, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+
+ for (; shift-- > tcount; ++pt)
+ {
+ *pt = ts >> (shift * 8);
+ }
+
+ for (i = end, b = point->val.int64; --i > start;)
+ {
+ point++;
+ ts = point->ts - (point - 1)->ts;
+
+ for (shift = tcount; shift--; ++pt)
+ {
+ *pt = ts >> (shift * 8);
+ }
+
+ if (vcount == 8)
+ {
+ memcpy(pt, &point->val.uint64, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ continue;
+ }
+
+ a = b;
+ b = point->val.int64;
+
+ if (a > b)
+ {
+ val = a - b;
+ val <<= 1;
+ val |= 1;
+ }
+ else
+ {
+ val = b - a;
+ val <<= 1;
+ };
+
+ for (shift = vcount; shift--; ++pt)
+ {
+ *pt = (val >> (shift * 8)) & 0xff;
+ }
+ }
+
+ return bits;
+}
+
+unsigned char * siridb_points_zip_string(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size)
+{
+ uint_fast32_t n = end - start;
+ if (n < POINTS_ZIP_THRESHOLD)
+ {
+ *size = sizeof(uint64_t);
+ return siridb_points_raw_string(points, start, end, cinfo, size);
+ }
+ size_t * sizes = malloc(sizeof(size_t) * n);
+ if (sizes == NULL)
+ {
+ return NULL;
+ }
+ uint8_t is_ascii = 0x80;
+ uint64_t tdiff = 0;
+ uint8_t * src, * out, * pt, * spt, * sout;
+ uint64_t mask;
+ size_t sz, m = n, i = end - 2;
+ uint32_t sz_src = 0;
+ siridb_point_t * point = points->data + i;
+ uint64_t ts = (point + 1)->ts - point->ts;
+ uint8_t tinfo = 0;
+ uint8_t shift = 0;
+ uint8_t ibit;
+
+ sz = POINTS_strlen_check_ascii((point + 1)->val.str, &is_ascii);
+ sizes[--m] = sz;
+ sz_src += sz;
+ sz = POINTS_strlen_check_ascii((point)->val.str, &is_ascii);
+ sizes[--m] = sz;
+ sz_src += sz;
+
+ while (i-- > start)
+ {
+ point--;
+ sz = POINTS_strlen_check_ascii(point->val.str, &is_ascii);
+ sizes[--m] = sz;
+ sz_src += sz;
+ tdiff |= ts ^ ((point + 1)->ts - point->ts);
+ }
+
+ for (i = 1, mask = 0xff; i <= 8; mask <<= 8, i++)
+ {
+ if (tdiff & mask)
+ {
+ tinfo = i;
+ }
+ else if (ts & mask)
+ {
+ shift = i;
+ }
+ }
+
+ shift = tinfo > shift ? tinfo : shift;
+ ibit = tinfo | (shift << 4);
+ /* calculate time-stamps size */
+ sz = 13 + shift + tinfo*(n - 2);
+
+ src = malloc(sz_src);
+ out = malloc(sz + sz_src + (is_ascii ? 0 : (n * 8)));
+ if (src == NULL || out == NULL)
+ {
+ goto failed;
+ }
+ pt = out;
+ sout = out + sz;
+ spt = src;
+
+ memcpy(pt, &ibit, sizeof(uint8_t));
+ pt++;
+ memcpy(pt, &sz_src, sizeof(uint32_t));
+ pt += sizeof(uint32_t);
+ memcpy(pt, &point->ts, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+
+ POINTS_zip_str(&sout, &spt, src, point->val.raw, sizes[m++], is_ascii);
+
+ for (; shift-- > tinfo; ++pt)
+ {
+ *pt = ts >> (shift * 8);
+ }
+ for (i = end; --i > start;)
+ {
+ point++;
+ ts = point->ts - (point - 1)->ts;
+
+ for (shift = tinfo; shift--; ++pt)
+ {
+ *pt = ts >> (shift * 8);
+ }
+
+ POINTS_zip_str(&sout, &spt, src, point->val.raw, sizes[m++], is_ascii);
+ }
+ sz = *size = sout - out;
+
+ if (POINTS_set_cinfo_size(cinfo, size))
+ {
+ goto failed;
+ }
+
+ if (*size > sz)
+ {
+ sout = (uint8_t *) realloc(out, *size);
+ if (sout == NULL)
+ {
+ goto failed;
+
+ }
+ pt = out = sout;
+ sout += *size;
+ pt += sz;
+ for (; pt < sout; ++pt)
+ {
+ *pt = '\0';
+ }
+ }
+
+ free(sizes);
+ free(src);
+ return out;
+
+failed:
+ free(sizes);
+ free(src);
+ free(out);
+ return NULL;
+}
+
+unsigned char * siridb_points_raw_string(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size)
+{
+ size_t ts_sz = *size;
+ *size = 0;
+
+ uint_fast32_t n = end - start;
+ size_t * sizes = malloc(sizeof(size_t) * n);
+ size_t * psz = sizes;
+ unsigned char * pdata;
+ unsigned char * cdata;
+ uint_fast32_t i;
+
+ if (sizes == NULL)
+ {
+ return NULL;
+ }
+
+ for (i = start; i < end; ++i, ++psz)
+ {
+ *psz = strlen(points->data[i].val.str) + 1;
+ *size += *psz;
+ }
+
+ if (POINTS_set_cinfo_size(cinfo, size))
+ {
+ free(sizes);
+ return NULL;
+ }
+
+ /* when uncompressed, time-stamp size is not included */
+ *size += n * ts_sz;
+
+ cdata = (unsigned char *) calloc(*size, sizeof(unsigned char));
+ if (cdata == NULL)
+ {
+ free(sizes);
+ return NULL;
+ }
+
+ pdata = cdata;
+ psz = sizes;
+
+ switch (ts_sz)
+ {
+ case sizeof(uint32_t):
+ for (i = start; i < end; ++i)
+ {
+ uint32_t ts = points->data[i].ts;
+ memcpy(pdata, &ts, sizeof(uint32_t));
+ pdata += sizeof(uint32_t);
+ }
+ break;
+ case sizeof(uint64_t):
+ for (i = start; i < end; ++i)
+ {
+ memcpy(pdata, &points->data[i].ts, sizeof(uint64_t));
+ pdata += sizeof(uint64_t);
+ }
+ break;
+ default:
+ assert(0);
+ }
+
+ for (i = start; i < end; ++i, ++psz)
+ {
+ memcpy(pdata, points->data[i].val.str, *psz);
+ pdata += *psz;
+ }
+
+ free(sizes);
+
+ return cdata;
+}
+/*
+ * Return NULL in case an error has occurred. This function destroys the
+ * original points.
+ */
+unsigned char * siridb_points_zip_double(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size)
+{
+ if (end - start < POINTS_ZIP_THRESHOLD)
+ {
+ return POINTS_zip_raw(points, start, end, cinfo, size);
+ }
+ uint64_t tdiff = 0;
+ uint64_t val;
+ size_t i = end - 2;
+ siridb_point_t * point = points->data + i;
+ uint64_t ts = (point + 1)->ts - point->ts;
+ uint64_t mask = (point + 1)->val.uint64;
+ uint64_t vdiff = mask ^ point->val.uint64;
+ uint8_t tcount = 0;
+ uint8_t vcount = 0;
+ uint8_t vstore = 0;
+ uint8_t shift = 0;
+ int vshift[RAW_VALUES_THRESHOLD];
+ unsigned char * bits, *pt;
+ int * pshift;
+
+ while (i-- > start)
+ {
+ point--;
+ vdiff |= mask ^ point->val.uint64;
+ tdiff |= ts ^ ((point + 1)->ts - point->ts);
+ }
+
+ for (i = 0, mask = 0xff; i < 8; mask <<= 8, i++)
+ {
+ if (vdiff & mask)
+ {
+ vstore |= 1 << i;
+ vshift[vcount++] = i * 8;
+ }
+ if (tdiff & mask)
+ {
+ tcount = i + 1;
+ }
+ else if (ts & mask)
+ {
+ shift = i + 1;
+ }
+ }
+ shift = (tcount > shift) ? tcount : shift;
+ *cinfo = vstore;
+ *cinfo <<= 8;
+ *cinfo |= tcount | (shift << 4);
+ *size = 16 + shift - tcount + (tcount + vcount) * (end - start - 1);
+ bits = malloc(*size);
+ if (bits == NULL)
+ {
+ return NULL;
+ }
+ pt = bits;
+
+ memcpy(pt, &point->ts, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ memcpy(pt, &point->val.uint64, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+
+ for (; shift-- > tcount; ++pt)
+ {
+ *pt = ts >> (shift * 8);
+ }
+
+ for (i = end; --i > start;)
+ {
+ point++;
+ ts = point->ts - (point - 1)->ts;
+ for (shift = tcount; shift--; ++pt)
+ {
+ *pt = ts >> (shift * 8);
+ }
+
+ val = point->val.uint64;
+
+ if (vcount > RAW_VALUES_THRESHOLD)
+ {
+ memcpy(pt, &val, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ continue;
+ }
+
+ for (pshift = vshift, shift = vcount; shift--; ++pshift, ++pt)
+ {
+ *pt = val >> *pshift;
+ }
+ }
+
+ return bits;
+}
+
+void siridb_points_unzip_int(
+ siridb_points_t * points,
+ unsigned char * bits,
+ uint16_t len,
+ uint16_t cinfo,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ if (len < POINTS_ZIP_THRESHOLD)
+ {
+ return POINTS_unzip_raw(
+ points, bits, len, start_ts, end_ts, has_overlap);
+ }
+ uint8_t vstore, tcount, tshift, j;
+ uint64_t ts, tmp, mask;
+ int64_t val;
+ size_t i;
+ unsigned char * pt = bits;
+ siridb_point_t * point = points->data + points->len;
+
+ vstore = cinfo >> 8;
+ tcount = cinfo & 0xf;
+ tshift = cinfo & 0xf0;
+ tshift >>= 4;
+
+ memcpy(&point->ts, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ memcpy(&point->val.uint64, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+
+ for (mask = 0; tshift-- > tcount; ++pt)
+ {
+ mask |= ((uint64_t) *pt) << (tshift * 8);
+ }
+
+ ts = point->ts;
+ val = point->val.int64;
+
+ for (i = len; (end_ts == NULL || ts < *end_ts) && --i; )
+ {
+ if (start_ts != NULL && ts < *start_ts)
+ {
+ --len;
+ }
+ else
+ {
+ ++point;
+ }
+
+ for (tmp = 0, j = tcount; j--; ++pt)
+ {
+ tmp <<= 8;
+ tmp |= *pt;
+ }
+ ts += tmp | mask;
+
+ point->ts = ts;
+
+ if (vstore != 0xff)
+ {
+ for (tmp = 0, j = vstore; j; j <<= 1, ++pt)
+ {
+ tmp <<= 8;
+ tmp |= *pt;
+ }
+ val += (tmp & 1) ? -(tmp >> 1) : (tmp >> 1);
+ point->val.int64 = val;
+ }
+ else
+ {
+ memcpy(&point->val.uint64, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ }
+ }
+
+ if (has_overlap && points->len)
+ {
+ qp_via_t v;
+ point = points->data + points->len;
+ for (i = len - i; i--; ++point)
+ {
+ ts = point->ts;
+ v = point->val;
+ siridb_points_add_point(points, &ts, &v);
+ }
+ }
+ else
+ {
+ points->len += len - i;
+ }
+}
+
+void siridb_points_unzip_double(
+ siridb_points_t * points,
+ unsigned char * bits,
+ uint16_t len,
+ uint16_t cinfo,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ if (len < POINTS_ZIP_THRESHOLD)
+ {
+ return POINTS_unzip_raw(
+ points, bits, len, start_ts, end_ts, has_overlap);
+ }
+ int vshift[RAW_VALUES_THRESHOLD];
+ int * pshift;
+ uint8_t vstore, tcount, tshift;
+ size_t i, c, j;
+ unsigned char * pt = bits;
+ uint64_t ts, tmp, mask, val;
+ siridb_point_t * point = points->data + points->len;
+
+ vstore = cinfo >> 8;
+ tcount = cinfo & 0xf;
+ tshift = cinfo & 0xf0;
+ tshift >>= 4;
+
+ for ( i = 0, c = 0, mask = 0, tmp = 0xff;
+ vstore;
+ vstore >>= 1, ++i, tmp <<= 8)
+ {
+ if (vstore & 1)
+ {
+ vshift[c++] = i * 8;
+ mask |= tmp;
+ }
+ }
+
+ memcpy(&point->ts, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ memcpy(&point->val.uint64, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+
+ ts = point->ts;
+ val = point->val.uint64 & ~mask;
+
+ for (mask = 0; tshift-- > tcount; ++pt)
+ {
+ mask |= ((uint64_t) *pt) << (tshift * 8);
+ }
+
+ for (i = len; (end_ts == NULL || ts < *end_ts) && --i; )
+ {
+ if (start_ts != NULL && ts < *start_ts)
+ {
+ --len;
+ }
+ else
+ {
+ ++point;
+ }
+
+ for (tmp = 0, j = tcount; j--; ++pt)
+ {
+ tmp <<= 8;
+ tmp |= *pt;
+ }
+ ts += mask | tmp;
+
+ point->ts = ts;
+
+ if (c > RAW_VALUES_THRESHOLD)
+ {
+ memcpy(&point->val.uint64, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ continue;
+ }
+
+ for (tmp = 0, pshift = vshift, j = c; j--; ++pshift, ++pt)
+ {
+ tmp |= ((uint64_t) *pt) << *pshift;
+ }
+ tmp |= val;
+ point->val.uint64 = tmp;
+ }
+
+ if (has_overlap && points->len)
+ {
+ qp_via_t v;
+ point = points->data + points->len;
+ for (i = len - i; i--; ++point)
+ {
+ ts = point->ts;
+ v = point->val;
+ siridb_points_add_point(points, &ts, &v);
+ }
+ }
+ else
+ {
+ points->len += len - i;
+ }
+}
+
+static size_t POINTS_dec_len(uint8_t **pt)
+{
+ size_t sz = 0;
+ uint8_t shift = 6;
+ uint8_t * b = *pt;
+
+ (*pt)++;
+ sz += *b & 0x3f;
+
+ if ((*b & 0x40) == 0)
+ {
+ return sz;
+ }
+
+ do
+ {
+ b = *pt;
+ (*pt)++;
+ sz |= (*b & 0x7f) << shift;
+ shift += 7;
+ } while ((*b & 0x80) != 0);
+
+ return sz;
+}
+
+static int POINTS_unpack_string(
+ siridb_point_t * point,
+ size_t n,
+ size_t offset,
+ uint8_t * src,
+ uint8_t * buf)
+{
+ assert (n);
+ uint8_t * pt = src;
+ uint8_t * sbuf = buf;
+ uint8_t is_ascii = (0x80 & (*pt)) ^ 0x80;
+ size_t i = 0;
+ while (1)
+ {
+ if (is_ascii ^ ((*pt) & 0x80))
+ {
+ /* literal */
+ size_t len = (is_ascii) ? 0 : POINTS_dec_len(&pt);
+
+ do
+ {
+ *buf = *pt;
+ ++pt;
+ if (!*buf)
+ {
+ if (i >= offset)
+ {
+ point->val.str = strdup((char *)sbuf);
+ if (point->val.str == NULL)
+ {
+ return -1;
+ }
+ ++point;
+ if (!--n)
+ {
+ return 0;
+ }
+ }
+ ++i;
+ ++buf;
+ sbuf = buf;
+ break;
+ }
+ ++buf;
+ }
+ while ((is_ascii && ((~(*pt)) & 0x80)) || ((!is_ascii) && --len));
+ }
+ else
+ {
+ /* match */
+ size_t off = POINTS_dec_len(&pt);
+ size_t len = POINTS_dec_len(&pt);
+ while (len--)
+ {
+ *buf = *(buf - off);
+ if (!*buf)
+ {
+ if (i >= offset)
+ {
+ point->val.str = strdup((char *)sbuf);
+ if (point->val.str == NULL)
+ {
+ return -1;
+ }
+ ++point;
+ if (!--n)
+ {
+ return 0;
+ }
+ }
+ ++i;
+ ++buf;
+ sbuf = buf;
+ }
+ else
+ {
+ ++buf;
+ }
+ }
+ }
+ }
+}
+
+int siridb_points_unzip_string_raw(
+ siridb_points_t * points,
+ uint8_t * bits,
+ uint16_t len)
+{
+ uint64_t * tpt;
+ tpt = (uint64_t *) bits;
+ char * cpt = (char *) (bits + (len * sizeof(uint64_t)));
+
+ for (; points->len < len; ++points->len, ++tpt)
+ {
+ size_t slen;
+ points->data[points->len].ts = *tpt;
+ points->data[points->len].val.str = xstr_dup(cpt, &slen);
+ cpt += slen + 1;
+ }
+ return 0;
+}
+
+int siridb_points_unzip_string(
+ siridb_points_t * points,
+ uint8_t * bits,
+ uint16_t len,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ uint64_t ts, tmp, mask;
+ siridb_point_t * point = points->data + points->len;
+ uint32_t src_sz;
+ uint8_t * buf, * pt = bits;
+ uint8_t j, tcount, tshift = *pt;
+ size_t i, offset;
+
+ tcount = *pt & 0xf;
+ tshift = *pt & 0xf0;
+ tshift >>= 4;
+ pt++;
+ offset = 13 + tshift + (tcount * (len - 2));
+
+ memcpy(&src_sz, pt, sizeof(uint32_t));
+ pt += sizeof(uint32_t);
+ memcpy(&point->ts, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+
+ buf = malloc(src_sz);
+ if (buf == NULL)
+ {
+ return -1;
+ }
+
+ for (mask = 0; tshift-- > tcount; ++pt)
+ {
+ mask |= ((uint64_t) *pt) << (tshift * 8);
+ }
+
+ ts = point->ts;
+ uint16_t n = len;
+
+ for (i = n; (end_ts == NULL || ts < *end_ts) && --i; )
+ {
+ if (start_ts != NULL && ts < *start_ts)
+ {
+ --n;
+ }
+ else
+ {
+ ++point;
+ }
+
+ for (tmp = 0, j = tcount; j--; ++pt)
+ {
+ tmp <<= 8;
+ tmp |= *pt;
+ }
+ ts += tmp | mask;
+
+ point->ts = ts;
+ }
+
+ n -= i;
+
+ if (POINTS_unpack_string(
+ points->data + points->len, n, i, bits + offset, buf))
+ {
+ free(buf);
+ return -1;
+ }
+
+ if (has_overlap && points->len)
+ {
+ qp_via_t v;
+ point = points->data + points->len;
+ for (i = n; i--; ++point)
+ {
+ ts = point->ts;
+ v = point->val;
+ siridb_points_add_point(points, &ts, &v);
+ }
+ }
+ else
+ {
+ points->len += n;
+ }
+
+ free(buf);
+ return 0;
+}
+
+size_t siridb_points_get_size_zipped(uint16_t cinfo, uint16_t len)
+{
+ if (len < POINTS_ZIP_THRESHOLD)
+ {
+ return len * 16;
+ }
+ uint8_t vcount = 0;
+ uint8_t vstore = cinfo >> 8;
+ uint8_t tcount = cinfo & 0xf;
+ uint8_t tshift = cinfo & 0xf0;
+ tshift >>= 4;
+
+ for (; vstore; vstore &= vstore-1)
+ {
+ vcount++;
+ }
+
+ return 16 + tshift - tcount + (tcount + vcount) * (len - 1);
+}
+
+static unsigned char * POINTS_zip_raw(
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ size_t * size)
+{
+ uint_fast32_t n = end - start;
+ uint_fast32_t i;
+ unsigned char * bits;
+ unsigned char * pt;
+
+ *size = n * 16;
+ *cinfo = 0xffff;
+
+ bits = malloc(*size);
+ if (bits == NULL)
+ {
+ return NULL;
+ }
+
+ pt = bits;
+ for (i = start; i < end; ++i)
+ {
+ memcpy(pt, &points->data[i].ts, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ memcpy(pt, &points->data[i].val, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ }
+
+ return bits;
+}
+
+static void POINTS_unzip_raw(
+ siridb_points_t * points,
+ unsigned char * bits,
+ uint16_t len,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ unsigned char * pt = bits;
+ siridb_point_t p;
+
+ while (len--)
+ {
+ memcpy(&p.ts, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ memcpy(&p.val, pt, sizeof(uint64_t));
+ pt += sizeof(uint64_t);
+ if ((start_ts == NULL || p.ts >= *start_ts) &&
+ (end_ts == NULL || p.ts < *end_ts))
+ {
+ if (has_overlap)
+ {
+ siridb_points_add_point(points, &p.ts, &p.val);
+ }
+ else
+ {
+ memcpy(points->data + points->len++, &p,
+ sizeof(siridb_point_t));
+ }
+ }
+ }
+}
+
+/*
+ * This merge method works best when having both a lot of series and having
+ * any number of points for each series.
+ */
+static void POINTS_highest_and_merge(vec_t * plist, siridb_points_t * points)
+{
+ siridb_points_t ** m = NULL;
+ size_t i, n, l = 0;
+ siridb_points_t ** tpts;
+ uint64_t highest = 0;
+ n = points->len;
+
+ while (plist->len)
+ {
+ if (m != NULL)
+ {
+ tpts = m;
+ m = NULL;
+ for (; l < plist->len; l++, tpts++)
+ {
+ if (((*tpts)->data + (*tpts)->len)->ts == highest)
+ {
+ m = tpts;
+ break;
+ }
+ }
+ }
+
+ if (m == NULL)
+ {
+ m = (siridb_points_t **) plist->data;
+
+ for (i = 1, tpts = m + 1; i < plist->len; i++, tpts++)
+ {
+ if (((*tpts)->data + (*tpts)->len)->ts >
+ ((*m)->data + (*m)->len)->ts)
+ {
+ m = tpts;
+ l = i;
+ highest = ((*m)->data + (*m)->len)->ts;
+ }
+ }
+ }
+
+ points->data[--n] = (*m)->data[(*m)->len];
+
+ if ((*m)->len--)
+ {
+ m++;
+ l++;
+ }
+ else
+ {
+ POINTS_destroy(*m);
+
+ if (--plist->len)
+ {
+ *m = (siridb_points_t *) plist->data[plist->len];
+ }
+ }
+
+ if (!(n % MAX_ITERATE_MERGE_COUNT))
+ {
+ usleep(3000);
+ }
+ }
+ /* size should be exactly zero */
+ assert (n == 0);
+}
+
+
+/*
+ * This merge method works best for only a few series with possible a lot
+ * of points.
+ */
+static void POINTS_sort_while_merge(vec_t * plist, siridb_points_t * points)
+{
+ siridb_points_t ** m;
+ size_t i, n;
+ siridb_points_t ** tpts;
+ n = points->len;
+
+ while (plist->len)
+ {
+ m = (siridb_points_t **) plist->data;
+
+ for (i = 1, tpts = m + 1; i < plist->len; i++, tpts++)
+ {
+ if (((*tpts)->data + (*tpts)->len)->ts >
+ ((*m)->data + (*m)->len)->ts)
+ {
+ m = tpts;
+ }
+ }
+
+ points->data[--n] = (*m)->data[(*m)->len];
+
+ if (!(*m)->len--)
+ {
+ POINTS_destroy(*m);
+
+ if (--plist->len)
+ {
+ *m = (siridb_points_t *) plist->data[plist->len];
+ }
+ }
+
+ if (!(n % MAX_ITERATE_MERGE_COUNT))
+ {
+ usleep(3000);
+ }
+ }
+ /* size should be exactly zero */
+ assert (n == 0);
+}
+
+/*
+ * This merge method works best when having a lot of series with only one point
+ * or when the total number of points it not so high.
+ */
+static void POINTS_merge_and_sort(vec_t * plist, siridb_points_t * points)
+{
+ siridb_points_t ** m;
+ size_t i, n;
+ n = points->len;
+ uint8_t use_simple_sort = n == plist->len;
+
+ while (plist->len)
+ {
+ for (i = 0; i < plist->len; i++)
+ {
+ m = (siridb_points_t **) &plist->data[i];
+
+ points->data[--n] = (*m)->data[(*m)->len];
+
+ if (!(*m)->len--)
+ {
+ POINTS_destroy(*m);
+ if (--plist->len)
+ {
+ *m = (siridb_points_t *) plist->data[plist->len];
+ }
+ }
+
+ if (!(n % MAX_ITERATE_MERGE_COUNT))
+ {
+ usleep(3000);
+ }
+ }
+ }
+ /* size should be exactly zero */
+ assert (n == 0);
+
+ usleep(5000);
+
+ if (use_simple_sort)
+ {
+ POINTS_simple_sort(points);
+ }
+ else
+ {
+ qsort( points->data,
+ points->len,
+ sizeof(siridb_point_t),
+ &POINTS_compare);
+ }
+}
+
+static inline int POINTS_compare(const void * a, const void * b)
+{
+ return (((siridb_point_t *) a)->ts < ((siridb_point_t *) b)->ts)
+ ? -1
+ : (((siridb_point_t *) a)->ts > ((siridb_point_t *) b)->ts)
+ ? 1
+ : 0;
+}
+
+/*
+ * Sort points by time-stamp.
+ *
+ * Warning: this function should only be used in another thread.
+ */
+static void POINTS_simple_sort(siridb_points_t * points)
+{
+ if (points->len < 2)
+ {
+ return; /* nothing to do... */
+ }
+ size_t n = 0;
+ size_t i;
+ siridb_point_t * pa, * pb;
+ siridb_point_t tmp;
+ while(++n < points->len)
+ {
+ pa = points->data + n - 1;
+ pb = points->data + n;
+ if (pa->ts > pb->ts)
+ {
+ tmp = points->data[n];
+ i = n;
+ do
+ {
+ points->data[i] = points->data[i - 1];
+ }
+ while (--i && points->data[i - 1].ts > tmp.ts);
+ points->data[i] = tmp;
+
+ usleep(100);
+ }
+ }
+}
+
+static size_t POINTS_strlen_check_ascii(const char * str, uint8_t * is_ascii)
+{
+ size_t sz;
+ if (*is_ascii)
+ {
+ const char * pt = str;
+ for (; *pt; ++pt)
+ {
+ *is_ascii &= (*pt) ^ 0x80;
+ }
+ sz = pt - str + 1;
+ }
+ else
+ {
+ sz = strlen(str) + 1;
+ }
+ return sz;
+}
+
+static void POINTS_output_literal(
+ size_t len,
+ uint8_t * pt,
+ uint8_t ** out,
+ uint8_t is_ascii)
+{
+ if (!is_ascii)
+ {
+ size_t n = len;
+ **out = 0x80 | ((n > 0x3f) << 6) | (n & 0x3f);
+ (*out)++;
+ n >>= 6;
+
+ while (n)
+ {
+ **out = ((n > 0x7f) << 7) | (n & 0x7f);
+ (*out)++;
+ n >>= 7;
+ }
+ }
+
+ while (len--)
+ {
+ **out = *pt;
+ (*out)++;
+ pt++;
+ }
+}
+
+static void POINTS_output_match(
+ size_t offset,
+ size_t len,
+ uint8_t ** out,
+ uint8_t is_ascii)
+{
+ **out = is_ascii | ((offset > 0x3f) << 6) | (offset & 0x3f);
+ (*out)++;
+ offset >>= 6;
+
+ while (offset)
+ {
+ **out = ((offset > 0x7f) << 7) | (offset & 0x7f);
+ (*out)++;
+ offset >>= 7;
+ }
+
+ **out = is_ascii | ((len > 0x3f) << 6) | (len & 0x3f);
+ (*out)++;
+ len >>= 6;
+
+ while (len)
+ {
+ **out = ((len > 0x7f) << 7) | (len & 0x7f);
+ (*out)++;
+ len >>= 7;
+ }
+}
+
+static void POINTS_zip_str(
+ uint8_t ** out,
+ uint8_t ** pt,
+ uint8_t * src,
+ uint8_t * s,
+ size_t n,
+ uint8_t is_ascii)
+{
+ uint8_t * match;
+ uint8_t * literal = *pt;
+ memcpy(*pt, s, n);
+ uint8_t * end = (*pt) + n;
+ uint8_t * tend = end - sizeof(uint32_t);
+ while (*pt < tend)
+ {
+ uint32_t * inp = (uint32_t *) *pt;
+ uint16_t idx = POINTS_hash(*inp);
+ match = dictionary[POINTS_hash(idx)];
+ dictionary[idx] = *pt;
+ if (match >= src && match < *pt && *((uint32_t *) match) == *inp)
+ {
+ if (literal < *pt)
+ {
+ POINTS_output_literal((*pt) - literal, literal, out, is_ascii);
+ }
+
+ uint8_t * p = (*pt) + 4;
+ match += 4;
+ while (p < end && *match == *p)
+ {
+ ++match;
+ ++p;
+ }
+
+ POINTS_output_match(p - match, p - (*pt), out, is_ascii);
+ literal = *pt = p;
+ }
+ else
+ {
+ ++(*pt);
+ }
+ }
+
+ if (literal < end)
+ {
+ POINTS_output_literal(end - literal, literal, out, is_ascii);
+ }
+ *pt = end;
+}
+
+static int POINTS_set_cinfo_size(uint16_t * cinfo, size_t * size)
+{
+ if (*size >= 0x8000)
+ {
+ if (*size > 0x2000000)
+ {
+ log_critical("Size too large: %zu", *size);
+ return -1;
+ }
+ *size = (!!((*size) & 0x3ff)) + ((*size) >> 10);
+ *cinfo = (*size) | 0x8000;
+ *size <<= 10;
+ }
+ else
+ {
+ *cinfo = *size;
+ }
+ return 0;
+}
+
+uint64_t siridb_points_get_interval(siridb_points_t * points)
+{
+ size_t i, j, n;
+ uint64_t * arr;
+ uint64_t x, a, b, c;
+
+ if (points->len < 8)
+ {
+ return 0;
+ }
+
+ n = points->len - 1;
+ n = n > 63 ? 63 : n;
+
+ arr = malloc(n * sizeof(uint64_t));
+ if (arr == NULL)
+ {
+ return 0;
+ }
+
+ for (i = 0; i < n; ++i)
+ {
+ x = points->data[i+1].ts - points->data[i].ts;
+ for (j = i; j > 0 && arr[j-1] > x; --j)
+ {
+ arr[j] = arr[j-1];
+ }
+ arr[j] = x;
+ }
+
+ a = n/4;
+ b = n/2;
+ c = arr[(b<<1)-a];
+ a = arr[a];
+ b = arr[b];
+ x = b / (100 / TOLERANCE_INTERVAL_DETECT);
+ x = (a+x < b || c-x > b) ? 0 : b;
+
+ free(arr);
+ return x;
+}
+
+inline static uint16_t POINTS_hash(uint32_t h)
+{
+ return ((h >> 17) ^ (h & 0xffff)) & DICT_SZ;
+}
+
+/*
+ * Destroy points. (parsing NULL is NOT allowed)
+ */
+static void POINTS_destroy(siridb_points_t * points)
+{
+ free(points->data);
+ free(points);
+}
--- /dev/null
+/*
+ * pool.c - SiriDB pool containing one or two servers.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/pool.h>
+#include <siri/db/pools.h>
+#include <siri/grammar/grammar.h>
+#include <stdlib.h>
+#include <string.h>
+#include <siri/db/server.h>
+
+
+/*
+ * Returns 1 (true) if at least one server in the pool is online, 0 (false)
+ * if no server in the pool is online.
+ *
+ * Warning: this function should not be used on 'this' pool.
+ *
+ * Server is 'online' when at least running and authenticated but not
+ * queue-full
+ */
+int siridb_pool_online(siridb_pool_t * pool)
+{
+ uint16_t i;
+ for (i = 0; i < pool->len; i++)
+ {
+ if (siridb_server_is_online(pool->server[i]))
+ {
+ return 1; /* true */
+ }
+ }
+ return 0; /* false */
+}
+
+/*
+ * Returns 1 (true) if at least one server in the pool is available, 0 (false)
+ * if no server in the pool is available.
+ *
+ * Warning: this function should not be used on 'this' pool.
+ *
+ * A server is 'available' when and ONLY when connected and authenticated.
+ */
+int siridb_pool_available(siridb_pool_t * pool)
+{
+ uint16_t i;
+ for (i = 0; i < pool->len; i++)
+ {
+ if (siridb_server_is_available(pool->server[i]))
+ {
+ return 1; /* true */
+ }
+ }
+ return 0; /* false */
+}
+
+/*
+ * Returns 1 (true) if at least one server in the pool is accessible, 0 (false)
+ * if no server in the pool is accessible.
+ *
+ * Warning: this function should not be used on 'this' pool.
+ *
+ * A server is 'accessible' when exactly 'available' or 're-indexing'.
+ */
+int siridb_pool_accessible(siridb_pool_t * pool)
+{
+ uint16_t i;
+ for (i = 0; i < pool->len; i++)
+ {
+ if (siridb_server_is_accessible(pool->server[i]))
+ {
+ return 1; /* true */
+ }
+ }
+ return 0; /* false */
+}
+
+
+/*
+ * Call-back function used to validate pools in a where expression.
+ *
+ * Returns 0 or 1 (false or true).
+ */
+int siridb_pool_cexpr_cb(
+ siridb_pool_walker_t * wpool,
+ cexpr_condition_t * cond)
+{
+ switch (cond->prop)
+ {
+ case CLERI_GID_K_POOL:
+ return cexpr_int_cmp(cond->operator, wpool->pool, cond->int64);
+ case CLERI_GID_K_SERVERS:
+ return cexpr_int_cmp(cond->operator, wpool->servers, cond->int64);
+ case CLERI_GID_K_SERIES:
+ return cexpr_int_cmp(cond->operator, wpool->series, cond->int64);
+ }
+ /* we must NEVER get here */
+ log_critical("Unexpected pool property received: %d", cond->prop);
+ assert (0);
+ return -1;
+}
+
+/*
+ * Add a server object to a pool.
+ */
+void siridb_pool_add_server(siridb_pool_t * pool, siridb_server_t * server)
+{
+ pool->len++;
+
+ if (pool->len == 1)
+ {
+ /* this is the first server found for this pool */
+ pool->server[0] = server;
+ pool->server[0]->id = 0;
+ }
+ else
+ {
+ /* we can only have 1 or 2 servers per pool */
+ assert (pool->len == 2);
+ /* add the server to the pool, ordered by UUID */
+ if (siridb_server_cmp(pool->server[0], server) < 0)
+ {
+ pool->server[1] = server;
+ pool->server[1]->id = 1;
+ }
+ else
+ {
+ assert (siridb_server_cmp(pool->server[0], server) > 0);
+ pool->server[1] = pool->server[0];
+ pool->server[0] = server;
+
+ /* set server id */
+ pool->server[0]->id = 0;
+ pool->server[1]->id = 1;
+ }
+ }
+}
+
+/*
+ * Returns 0 if we have send the package to one 'accessible'
+ * server in the pool. The package will be send only to one server, even if
+ * the pool has more servers 're-indexing'.
+ *
+ * This function can raise a SIGNAL when allocation errors occur but be aware
+ * that 0 can still be returned in this case.
+ *
+ * The call-back function is not called when -1 is returned.
+ *
+ * A server is 're-indexing' when exactly running and authenticated and
+ * optionally re-indexing.
+ *
+ * Note that 'pkg->pid' will be overwritten with a new package id.
+ *
+ * In case flag 'FLAG_ONLY_CHECK_ONLINE' is set, we do not check 'accessible'
+ * but only 'online' is enough.
+ *
+ * pkg will be destroyed when and ONLY when 0 is returned. (Except when
+ * FLAG_KEEP_PKG is set)
+ */
+int siridb_pool_send_pkg(
+ siridb_pool_t * pool,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promise_cb cb,
+ void * data,
+ int flags)
+{
+ siridb_server_t * server = NULL;
+ uint16_t i;
+
+ for (i = 0; i < pool->len; i++)
+ {
+ if ((flags & FLAG_ONLY_CHECK_ONLINE) ?
+ siridb_server_is_online(pool->server[i]) :
+ siridb_server_is_accessible(pool->server[i]))
+ {
+ server = (server == NULL) ?
+ pool->server[i] : pool->server[rand() % 2];
+ }
+ }
+
+ return (server == NULL) ?
+ -1: siridb_server_send_pkg(server, pkg, timeout, cb, data, flags);
+}
--- /dev/null
+/*
+ * pools.c - Collection of pools.
+ */
+#include <assert.h>
+#include <llist/llist.h>
+#include <logger/logger.h>
+#include <siri/db/pools.h>
+#include <siri/db/server.h>
+#include <siri/net/promises.h>
+#include <siri/optimize.h>
+#include <stdlib.h>
+#include <string.h>
+#include <siri/err.h>
+
+static int POOLS_max_pool(siridb_server_t * server, uint16_t * max_pool);
+static int POOLS_arrange(siridb_server_t * server, siridb_t * siridb);
+
+/*
+ * This function can raise an ALLOC signal.
+ */
+void siridb_pools_init(siridb_t * siridb)
+{
+ assert (siridb->pools == NULL);
+ assert (siridb->servers != NULL && siridb->servers->len > 0);
+ assert (siridb->server != NULL);
+
+ siridb->pools = malloc(sizeof(siridb_pools_t));
+ if (siridb->pools == NULL)
+ {
+ ERR_ALLOC
+ return;
+ }
+ uint16_t max_pool = 0;
+ uint16_t n;
+
+ /* get max_pool (this can be used to get the number of pools) */
+ llist_walk(siridb->servers, (llist_cb) POOLS_max_pool, &max_pool);
+
+ /* set number of pools */
+ siridb->pools->len = max_pool + 1;
+
+ /* allocate memory for all pools */
+ siridb->pools->pool = malloc(sizeof(siridb_pool_t) * siridb->pools->len);
+
+ if (siridb->pools->pool == NULL)
+ {
+ ERR_ALLOC
+ free(siridb->pools);
+ siridb->pools = NULL;
+ return;
+ }
+
+ /* initialize number of servers with zero for each pool */
+ for (n = 0; n < siridb->pools->len; n++)
+ {
+ siridb->pools->pool[n].len = 0;
+ }
+
+ /* signal can be raised if creating a fifo buffer fails */
+ llist_walk(siridb->servers, (llist_cb) POOLS_arrange, siridb);
+
+ siridb->pools->prev_lookup = NULL;
+
+ /* generate pool lookup for series */
+ siridb->pools->lookup = siridb_lookup_new(siridb->pools->len);
+ if (siridb->pools->lookup == NULL)
+ {
+ siridb_pools_free(siridb->pools);
+ siridb->pools = NULL;
+ /* signal is raised */
+ }
+}
+
+/*
+ * Destroy pools. (parsing NULL is NOT allowed)
+ */
+void siridb_pools_free(siridb_pools_t * pools)
+{
+ free(pools->pool);
+ siridb_lookup_free(pools->lookup);
+ siridb_lookup_free(pools->prev_lookup);
+ free(pools);
+}
+
+/*
+ * Append a new pool to pools and returns the new created pool object.
+ *
+ * Note: pools->prev_lookup is set to the previous lookup table and a new
+ * lookup is created and set to pools->lookup.
+ *
+ * Returns NULL and raises a SIGNAL in case an error has occurred and pools
+ * remains unchanged in this case.
+ */
+siridb_pool_t * siridb_pools_append(
+ siridb_pools_t * pools,
+ siridb_server_t * server)
+{
+ siridb_pool_t * pool = NULL;
+ siridb_lookup_t * lookup = siridb_lookup_new(pools->len + 1);
+ if (lookup != NULL)
+ {
+ pool = (siridb_pool_t *)
+ realloc(pools->pool, sizeof(siridb_pool_t) * (pools->len + 1));
+ if (pool == NULL)
+ {
+ ERR_ALLOC
+ siridb_lookup_free(lookup);
+ }
+ else
+ {
+ pools->pool = pool;
+ pool = &pools->pool[pools->len];
+ pool->len = 0;
+ siridb_pool_add_server(pool, server);
+ pools->len++;
+
+ assert (pools->prev_lookup == NULL);
+
+ pools->prev_lookup = pools->lookup;
+ pools->lookup = lookup;
+ }
+ }
+ return pool;
+}
+
+
+/*
+ * Returns 1 (true) if at least one server in each pool is online, 0 (false)
+ * if at least one pool has no server online. ('this' pool is NOT included)
+ *
+ * A server is considered 'online' when connected and authenticated.
+ */
+int siridb_pools_online(siridb_t * siridb)
+{
+ uint16_t pid ;
+ for (pid = 0; pid < siridb->pools->len; pid++)
+ {
+ if ( pid != siridb->server->pool &&
+ !siridb_pool_online(siridb->pools->pool + pid))
+ {
+ return 0; /* false */
+ }
+ }
+ return 1; /* true */
+}
+
+/*
+ * Returns 1 (true) if at least one server in each pool is available, 0 (false)
+ * if at least one pool has no server available. ('this' pool is NOT included)
+ *
+ * A server is 'available' when and ONLY when connected and authenticated.
+ */
+int siridb_pools_available(siridb_t * siridb)
+{
+ uint16_t pid;
+ for (pid = 0; pid < siridb->pools->len; pid++)
+ {
+ if ( pid != siridb->server->pool &&
+ !siridb_pool_available(siridb->pools->pool + pid))
+ {
+ return 0; /* false */
+ }
+ }
+ return 1; /* true */
+}
+
+
+/*
+ * Returns 1 (true) if at least one server in each pool is accessible, 0 (false)
+ * if at least one pool has no server accessible. ('this' pool is NOT included)
+ *
+ * A server is 'accessible' when exactly 'available' or 're-indexing'.
+ */
+int siridb_pools_accessible(siridb_t * siridb)
+{
+ uint16_t pid;
+ for (pid = 0; pid < siridb->pools->len; pid++)
+ {
+ if ( pid != siridb->server->pool &&
+ !siridb_pool_accessible(siridb->pools->pool + pid))
+ {
+ return 0; /* false */
+ }
+ }
+ return 1; /* true */
+}
+
+/*
+ * This function will send a package to one accessible server in each pool,
+ * 'this' pool not included. The promises call-back function should be
+ * used to check if the package has been send successfully to all pools.
+ *
+ * This function can raise a SIGNAL when allocation errors occur.
+ *
+ * If promises could not be created, the 'cb' function with cb(NULL, data)
+ *
+ * Note that 'pkg->pid' will be overwritten with a new package id and pkg
+ * will always be destroyed.
+ */
+void siridb_pools_send_pkg(
+ siridb_t * siridb,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promises_cb cb,
+ void * data,
+ int flags)
+{
+ sirinet_promises_t * promises =
+ sirinet_promises_new(siridb->pools->len - 1, cb, data, pkg);
+
+ if (promises == NULL)
+ {
+ free(pkg);
+ cb(NULL, data);
+ }
+ else
+ {
+ sirinet_pkg_t * dup;
+ siridb_pool_t * pool;
+ uint16_t pid;
+
+ for (pid = 0; pid < siridb->pools->len; pid++)
+ {
+ if (pid == siridb->server->pool)
+ {
+ continue;
+ }
+
+ pool = siridb->pools->pool + pid;
+
+ if ((dup = sirinet_pkg_dup(pkg)) == NULL ||
+ siridb_pool_send_pkg(
+ pool,
+ dup,
+ timeout,
+ (sirinet_promise_cb) sirinet_promises_on_response,
+ promises,
+ flags & ~FLAG_KEEP_PKG))
+ {
+ log_debug(
+ "Cannot send package to pool '%u' "
+ "(no accessible server found)",
+ pid);
+ free(dup);
+ vec_append(promises->promises, NULL);
+ }
+ }
+
+ SIRINET_PROMISES_CHECK(promises)
+ }
+}
+
+/*
+ * This function will send a package to one accessible server in the pools
+ * provided by the 'vec'. The promises call-back function should be
+ * used to check if the package has been send successfully to all pools.
+ *
+ * The 'vec' can be destroyed after this function has returned.
+ *
+ * This function can raise a SIGNAL when allocation errors occur.
+ *
+ * If promises could not be created, the 'cb' function with cb(NULL, data)
+ *
+ * Note that 'pkg->pid' will be overwritten with a new package id and pkg
+ * will always be destroyed.
+ */
+void siridb_pools_send_pkg_2some(
+ vec_t * vec,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promises_cb cb,
+ void * data,
+ int flags)
+{
+ sirinet_promises_t * promises =
+ sirinet_promises_new(vec->len, cb, data, pkg);
+
+ if (promises == NULL)
+ {
+ free(pkg);
+ cb(NULL, data);
+ }
+ else
+ {
+ sirinet_pkg_t * dup;
+ siridb_pool_t * pool;
+ size_t i;
+
+ for (i = 0; i < vec->len; i++)
+ {
+ pool = vec->data[i];
+
+ if ((dup = sirinet_pkg_dup(pkg)) == NULL ||
+ siridb_pool_send_pkg(
+ pool,
+ dup,
+ timeout,
+ (sirinet_promise_cb) sirinet_promises_on_response,
+ promises,
+ flags & ~FLAG_KEEP_PKG))
+ {
+ log_debug(
+ "Cannot send package to at least on pool "
+ "(no accessible server found)");
+ free(dup);
+ vec_append(promises->promises, NULL);
+ }
+ }
+
+ SIRINET_PROMISES_CHECK(promises)
+ }
+}
+
+
+static int POOLS_max_pool(siridb_server_t * server, uint16_t * max_pool)
+{
+ if (server->pool > *max_pool)
+ {
+ *max_pool = server->pool;
+ }
+ return 0;
+}
+
+/*
+ * Signal can be raised by this function when a fifo buffer for an optional
+ * replica server can't be created.
+ */
+static int POOLS_arrange(siridb_server_t * server, siridb_t * siridb)
+{
+ siridb_pool_t * pool = siridb->pools->pool + server->pool;
+
+ /* if the server is member of the same pool, it must be the replica */
+ if (siridb->server != server && siridb->server->pool == server->pool)
+ {
+ siridb->replica = server;
+
+ /* set synchronize flag */
+ siridb->server->flags |= SERVER_FLAG_SYNCHRONIZING;
+
+ /* pause optimize task while synchronize flag is set */
+ siri_optimize_pause();
+
+ /* initialize replica */
+ siridb->fifo = siridb_fifo_new(siridb);
+
+ if (siridb->fifo == NULL)
+ {
+ log_critical("Cannot initialize fifo buffer for replica server");
+ /* signal is set */
+ }
+ else
+ {
+ /* signal can be raised by 'siridb_replicate_init' */
+ siridb_replicate_init(
+ siridb,
+ siridb_initsync_open(siridb, 0));
+ }
+ }
+
+ siridb_pool_add_server(pool, server);
+ return 0;
+}
--- /dev/null
+/*
+ * presuf.c - Prefix and Suffix store.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/presuf.h>
+#include <siri/err.h>
+#include <siri/grammar/grammar.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <xstr/xstr.h>
+#include <string.h>
+
+siridb_presuf_t * PRESUF_add(siridb_presuf_t ** presuf);
+int PRESUF_is_equal(const char * a, const char * b);
+
+static char * presuffed_name = NULL;
+static size_t presuffed_size = 0;
+
+/*
+ * Memory will be allocated for 'presuffed_name' using malloc.
+ * This function must be called once to free the memory.
+ */
+void siridb_presuf_cleanup(void)
+{
+ free(presuffed_name);
+ presuffed_name = NULL;
+ presuffed_size = 0;
+}
+
+/*
+ * Destroy prefix-suffix object.
+ */
+void siridb_presuf_free(siridb_presuf_t * presuf)
+{
+ siridb_presuf_t * tmp;
+ while (presuf != NULL)
+ {
+ tmp = presuf;
+ presuf = presuf->prev;
+ free(tmp->prefix);
+ free(tmp->suffix);
+ free(tmp);
+ }
+}
+
+/*
+ * Add prefix-suffix from a node.
+ * Use 'siridb_presuf_is_unique' to check if the new node is unique.
+ *
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ * (presuf remains unchanged in case of an error)
+ */
+siridb_presuf_t * siridb_presuf_add(
+ siridb_presuf_t ** presuf,
+ cleri_node_t * node)
+{
+ cleri_children_t * children = node->children->next;
+ cleri_children_t * ps_children;
+
+ siridb_presuf_t * nps = PRESUF_add(presuf);
+ if (nps != NULL)
+ {
+ while (children != NULL)
+ {
+ ps_children = children->node->children->node->children;
+ switch (ps_children->node->cl_obj->gid)
+ {
+ case CLERI_GID_K_PREFIX:
+ nps->prefix = malloc(ps_children->next->node->len + 1);
+ if (nps->prefix != NULL)
+ {
+ /* not critical if suffix is still NULL */
+ nps->len += xstr_extract_string(
+ nps->prefix,
+ ps_children->next->node->str,
+ ps_children->next->node->len);
+ }
+ break;
+ case CLERI_GID_K_SUFFIX:
+ nps->suffix = malloc(ps_children->next->node->len + 1);
+ if (nps->suffix != NULL)
+ {
+ /* not critical if suffix is still NULL */
+ nps->len += xstr_extract_string(
+ nps->suffix,
+ ps_children->next->node->str,
+ ps_children->next->node->len);
+ }
+ break;
+ default:
+ assert (0);
+ break;
+ }
+ children = children->next;
+ }
+ }
+ return nps;
+}
+
+/*
+ * Check if the last presuf is unique compared to the others.
+ */
+int siridb_presuf_is_unique(siridb_presuf_t * presuf)
+{
+ const char * prefix = presuf->prefix;
+ const char * suffix = presuf->suffix;
+ while (presuf->prev != NULL)
+ {
+ presuf = presuf->prev;
+ if ( PRESUF_is_equal(prefix, presuf->prefix) &&
+ PRESUF_is_equal(suffix, presuf->suffix))
+ {
+ return 0; /* false, not unique */
+ }
+ }
+ return 1; /* true, unique */
+}
+
+const char * siridb_presuf_name(
+ siridb_presuf_t * presuf,
+ const char * name,
+ size_t len)
+{
+ size_t size = len + presuf->len;
+ if (size > presuffed_size)
+ {
+ char * tmp;
+ tmp = (char *) realloc(presuffed_name, size);
+ if (tmp == NULL)
+ {
+ ERR_ALLOC
+ return NULL;
+ }
+ presuffed_size = size;
+ presuffed_name = tmp;
+ }
+ sprintf(presuffed_name,
+ "%s%s%s",
+ (presuf->prefix == NULL) ? "" : presuf->prefix,
+ name,
+ (presuf->suffix == NULL) ? "" : presuf->suffix);
+ return presuffed_name;
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ *
+ * In case of an error, the original presuf object remains unchanged,
+ * when successful the returned object is the same as the presuf object.
+ */
+siridb_presuf_t * PRESUF_add(siridb_presuf_t ** presuf)
+{
+ siridb_presuf_t * newps = malloc(sizeof(siridb_presuf_t));
+ if (newps == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ newps->prefix = NULL;
+ newps->suffix = NULL;
+ newps->len = 1; /* we include the terminator char in the length */
+ newps->prev = *presuf;
+ *presuf = newps;
+ }
+
+ return newps;
+}
+
+/*
+ * Like 'strcmp' but also accepts NULL.
+ */
+int PRESUF_is_equal(const char * a, const char * b)
+{
+ if ((a == NULL) ^ (b == NULL))
+ {
+ return 0;
+ }
+ return (a == NULL) || (strcmp(a, b) == 0);
+}
--- /dev/null
+/*
+ * props.c - Functions to return SiriDB properties.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <procinfo/procinfo.h>
+#include <siri/db/initsync.h>
+#include <siri/db/props.h>
+#include <siri/db/reindex.h>
+#include <siri/db/time.h>
+#include <siri/grammar/grammar.h>
+#include <siri/db/fifo.h>
+#include <siri/db/tee.h>
+#include <siri/net/tcp.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <stdio.h>
+#include <string.h>
+#include <uuid/uuid.h>
+#include <uv.h>
+
+#define SIRIDB_PROP_MAP(NAME, LEN) \
+if (map) \
+{ \
+ qp_add_type(packer, QP_MAP2); \
+ qp_add_raw(packer, (const unsigned char *) "name", 4); \
+ qp_add_raw(packer, (const unsigned char *) NAME, LEN); \
+ qp_add_raw(packer, (const unsigned char *) "value", 5); \
+}
+
+static siridb_props_cb SIRIDB_PROPS[KW_COUNT];
+
+siridb_props_cb props_get_cb(int i)
+{
+ return SIRIDB_PROPS[i];
+}
+
+static void props_set_cb(int i, siridb_props_cb cb)
+{
+ SIRIDB_PROPS[i] = cb;
+}
+
+static char * WHO_AM_I = NULL;
+
+void props_set_who_am_i(char * s)
+{
+ WHO_AM_I = s;
+}
+
+char * props_get_who_am_i(void)
+{
+ return WHO_AM_I;
+}
+
+static void prop_active_handles(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_active_tasks(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_buffer_path(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_buffer_size(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_dbname(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_dbpath(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_drop_threshold(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_duration_log(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_duration_num(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_expiration_log(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_expiration_num(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_fifo_files(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_idle_percentage(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_idle_time(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_ip_support(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_libuv(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_list_limit(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_log_level(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_max_open_files(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_mem_usage(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_open_files(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_pool(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_received_points(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_reindex_progress(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_selected_points(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_select_points_limit(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_server(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_startup_time(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_status(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_sync_progress(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_tee_pipe_name(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_timezone(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_time_precision(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_uptime(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_uuid(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_version(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+static void prop_who_am_i(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map);
+
+void siridb_init_props(void)
+{
+ uint_fast16_t i;
+
+ for (i = 0; i < KW_COUNT; i++)
+ {
+ props_set_cb(i, NULL);
+ }
+
+ props_set_cb(CLERI_GID_K_ACTIVE_HANDLES - KW_OFFSET,
+ prop_active_handles);
+ props_set_cb(CLERI_GID_K_ACTIVE_TASKS - KW_OFFSET,
+ prop_active_tasks);
+ props_set_cb(CLERI_GID_K_BUFFER_PATH - KW_OFFSET,
+ prop_buffer_path);
+ props_set_cb(CLERI_GID_K_BUFFER_SIZE - KW_OFFSET,
+ prop_buffer_size);
+ props_set_cb(CLERI_GID_K_DBNAME - KW_OFFSET,
+ prop_dbname);
+ props_set_cb(CLERI_GID_K_DBPATH - KW_OFFSET,
+ prop_dbpath);
+ props_set_cb(CLERI_GID_K_DROP_THRESHOLD - KW_OFFSET,
+ prop_drop_threshold);
+ props_set_cb(CLERI_GID_K_DURATION_LOG - KW_OFFSET,
+ prop_duration_log);
+ props_set_cb(CLERI_GID_K_DURATION_NUM - KW_OFFSET,
+ prop_duration_num);
+ props_set_cb(CLERI_GID_K_FIFO_FILES - KW_OFFSET,
+ prop_fifo_files);
+ props_set_cb(CLERI_GID_K_EXPIRATION_LOG - KW_OFFSET,
+ prop_expiration_log);
+ props_set_cb(CLERI_GID_K_EXPIRATION_NUM - KW_OFFSET,
+ prop_expiration_num);
+ props_set_cb(CLERI_GID_K_IDLE_PERCENTAGE - KW_OFFSET,
+ prop_idle_percentage);
+ props_set_cb(CLERI_GID_K_IDLE_TIME - KW_OFFSET,
+ prop_idle_time);
+ props_set_cb(CLERI_GID_K_IP_SUPPORT - KW_OFFSET,
+ prop_ip_support);
+ props_set_cb(CLERI_GID_K_LIBUV - KW_OFFSET,
+ prop_libuv);
+ props_set_cb(CLERI_GID_K_LIST_LIMIT - KW_OFFSET,
+ prop_list_limit);
+ props_set_cb(CLERI_GID_K_MAX_OPEN_FILES - KW_OFFSET,
+ prop_max_open_files);
+ props_set_cb(CLERI_GID_K_MEM_USAGE - KW_OFFSET,
+ prop_mem_usage);
+ props_set_cb(CLERI_GID_K_LOG_LEVEL - KW_OFFSET,
+ prop_log_level);
+ props_set_cb(CLERI_GID_K_OPEN_FILES - KW_OFFSET,
+ prop_open_files);
+ props_set_cb(CLERI_GID_K_POOL - KW_OFFSET,
+ prop_pool);
+ props_set_cb(CLERI_GID_K_RECEIVED_POINTS - KW_OFFSET,
+ prop_received_points);
+ props_set_cb(CLERI_GID_K_REINDEX_PROGRESS - KW_OFFSET,
+ prop_reindex_progress);
+ props_set_cb(CLERI_GID_K_SELECTED_POINTS - KW_OFFSET,
+ prop_selected_points);
+ props_set_cb(CLERI_GID_K_SELECT_POINTS_LIMIT - KW_OFFSET,
+ prop_select_points_limit);
+ props_set_cb(CLERI_GID_K_SERVER - KW_OFFSET,
+ prop_server);
+ props_set_cb(CLERI_GID_K_STARTUP_TIME - KW_OFFSET,
+ prop_startup_time);
+ props_set_cb(CLERI_GID_K_STATUS - KW_OFFSET,
+ prop_status);
+ props_set_cb(CLERI_GID_K_SYNC_PROGRESS - KW_OFFSET,
+ prop_sync_progress);
+ props_set_cb(CLERI_GID_K_TEE_PIPE_NAME - KW_OFFSET,
+ prop_tee_pipe_name);
+ props_set_cb(CLERI_GID_K_TIMEZONE - KW_OFFSET,
+ prop_timezone);
+ props_set_cb(CLERI_GID_K_TIME_PRECISION - KW_OFFSET,
+ prop_time_precision);
+ props_set_cb(CLERI_GID_K_UPTIME - KW_OFFSET,
+ prop_uptime);
+ props_set_cb(CLERI_GID_K_UUID - KW_OFFSET,
+ prop_uuid);
+ props_set_cb(CLERI_GID_K_VERSION - KW_OFFSET,
+ prop_version);
+ props_set_cb(CLERI_GID_K_WHO_AM_I - KW_OFFSET,
+ prop_who_am_i);
+}
+
+static void prop_active_handles(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("active_handles", 14)
+ qp_add_int64(packer, (int64_t) siri.loop->active_handles);
+}
+
+static void prop_active_tasks(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("active_tasks", 12)
+ qp_add_int64(packer, (int64_t) siridb->tasks.active);
+}
+
+static void prop_buffer_path(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("buffer_path", 11)
+ qp_add_string(packer, siridb->buffer->path);
+}
+
+static void prop_buffer_size(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("buffer_size", 11)
+ qp_add_int64(packer, (int64_t) siridb->buffer->size);
+}
+
+static void prop_dbname(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("dbname", 6)
+ qp_add_string(packer, siridb->dbname);
+}
+
+static void prop_dbpath(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("dbpath", 6)
+ qp_add_string(packer, siridb->dbpath);
+}
+
+static void prop_drop_threshold(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("drop_threshold", 14)
+ qp_add_double(packer, siridb->drop_threshold);
+}
+
+static void prop_duration_log(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("duration_log", 12)
+ qp_add_int64(packer, (int64_t) siridb->duration_log);
+}
+
+static void prop_duration_num(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("duration_num", 12)
+ qp_add_int64(packer, (int64_t) siridb->duration_num);
+}
+
+static void prop_fifo_files(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("fifo_files", 10)
+ qp_add_int64(packer, (int64_t) siridb_fifo_size(siridb->fifo));
+}
+
+static void prop_expiration_log(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("expiration_log", 14)
+ if (siridb->expiration_log)
+ {
+ qp_add_int64(packer, (int64_t) siridb->expiration_log);
+ }
+ else
+ {
+ qp_add_null(packer);
+ }
+}
+
+static void prop_expiration_num(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("expiration_num", 14)
+ if (siridb->expiration_num)
+ {
+ qp_add_int64(packer, (int64_t) siridb->expiration_num);
+ }
+ else
+ {
+ qp_add_null(packer);
+ }
+}
+
+static void prop_idle_percentage(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("idle_percentage", 15)
+ qp_add_int64(packer, siridb_get_idle_percentage(siridb));
+}
+
+static void prop_idle_time(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("idle_time", 9)
+ qp_add_int64(packer, (int64_t) siridb->tasks.idle_time);
+}
+
+static void prop_ip_support(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("ip_support", 10)
+ qp_add_string(packer, sirinet_tcp_ip_support_str(siri.cfg->ip_support));
+}
+
+static void prop_libuv(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("libuv", 5)
+ qp_add_string(packer, uv_version_string());
+}
+
+static void prop_list_limit(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("list_limit", 10)
+ qp_add_int64(packer, (int64_t) siridb->list_limit);
+}
+
+static void prop_log_level(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("log_level", 9)
+ qp_add_string(packer, Logger.level_name);
+}
+
+static void prop_max_open_files(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("max_open_files", 14)
+ qp_add_int64(packer, (int64_t) siri.cfg->max_open_files);
+}
+
+static void prop_mem_usage(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("mem_usage", 9)
+ qp_add_int64(packer, (int64_t) (procinfo_total_physical_memory() / 1024));
+}
+
+static void prop_open_files(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("open_files", 10)
+ qp_add_int64(packer, (int64_t) siridb_open_files(siridb));
+}
+
+static void prop_pool(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("pool", 4)
+ qp_add_int64(packer, (int64_t) siridb->server->pool);
+}
+
+static void prop_received_points(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("received_points", 15)
+ qp_add_int64(packer, (int64_t) siridb->received_points);
+}
+
+static void prop_reindex_progress(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("reindex_progress", 16)
+ qp_add_string(packer, siridb_reindex_progress(siridb));
+}
+
+static void prop_selected_points(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("selected_points", 15)
+ qp_add_int64(packer, (int64_t) siridb->selected_points);
+}
+
+static void prop_select_points_limit(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("select_points_limit", 19)
+ qp_add_int64(packer, (int64_t) siridb->select_points_limit);
+}
+
+static void prop_server(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("server", 6)
+ qp_add_string(packer, siridb->server->name);
+}
+
+static void prop_startup_time(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("startup_time", 12)
+ qp_add_int64(packer, (int64_t) siri.startup_time);
+}
+
+static void prop_status(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("status", 6)
+ char * status = siridb_server_str_status(siridb->server);
+ qp_add_string(packer, status);
+ free(status);
+}
+
+static void prop_sync_progress(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("sync_progress", 13)
+ qp_add_string(packer, siridb_initsync_sync_progress(siridb));
+}
+
+static void prop_tee_pipe_name(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("tee_pipe_name", 13)
+ qp_add_string(packer, tee_str(siridb->tee));
+}
+
+static void prop_timezone(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("timezone", 8)
+ qp_add_string(packer, iso8601_tzname(siridb->tz));
+}
+
+static void prop_time_precision(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("time_precision", 14)
+
+ assert (siridb->time->precision >= SIRIDB_TIME_SECONDS &&
+ siridb->time->precision <= SIRIDB_TIME_NANOSECONDS);
+
+ qp_add_string(packer, siridb_time_short_map(siridb->time->precision));
+}
+
+static void prop_uptime(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("uptime", 6)
+ qp_add_int64(packer, siridb_get_uptime(siridb));
+}
+
+static void prop_uuid(
+ siridb_t * siridb,
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("uuid", 4)
+ char uuid[37];
+ uuid_unparse_lower(siridb->uuid, uuid);
+ qp_add_string(packer, uuid);
+}
+
+static void prop_version(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("version", 7)
+ qp_add_string(packer, SIRIDB_VERSION);
+}
+
+static void prop_who_am_i(
+ siridb_t * siridb __attribute__((unused)),
+ qp_packer_t * packer,
+ int map)
+{
+ SIRIDB_PROP_MAP("who_am_i", 8)
+ qp_add_string(packer, props_get_who_am_i());
+}
--- /dev/null
+/*
+ * queries.c - Query helpers for listener.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/aggregate.h>
+#include <siri/db/query.h>
+#include <siri/db/shard.h>
+#include <siri/db/queries.h>
+#include <siri/db/sset.h>
+#include <stddef.h>
+#include <stdlib.h>
+
+#define DEFAULT_LIST_LIMIT 1000
+
+#define QUERIES_NEW(q) \
+q->flags = 0; \
+q->series_map = NULL; \
+q->series_tmp = NULL; \
+q->sset_vec = NULL; \
+q->vec = NULL; \
+q->vec_index = 0; \
+q->pmap = NULL; \
+q->update_cb = NULL; \
+q->where_expr = NULL; \
+q->regex = NULL; \
+q->match_data = NULL;
+
+
+#define QUERIES_FREE(q, handle) \
+vec_destroy(q->sset_vec, (vec_destroy_cb) siridb_sset_free); \
+if (q->series_map != NULL) \
+{ \
+ imap_free( \
+ q->series_map, \
+ (imap_free_cb) &siridb__series_decref); \
+} \
+if (q->series_tmp != NULL) \
+{ \
+ imap_free( \
+ q->series_tmp, \
+ (imap_free_cb) &siridb__series_decref); \
+} \
+if (q->vec != NULL) \
+{ \
+ siridb_series_t * series; \
+ for (; q->vec_index < q->vec->len; q->vec_index++) \
+ { \
+ series = (siridb_series_t *) q->vec->data[q->vec_index]; \
+ siridb_series_decref(series); \
+ } \
+ vec_free(q->vec); \
+} \
+if (q->where_expr != NULL) \
+{ \
+ cexpr_free(q->where_expr); \
+} \
+if (q->pmap != NULL) \
+{ \
+ imap_free(q->pmap, NULL); \
+} \
+pcre2_code_free(q->regex); \
+pcre2_match_data_free(q->match_data); \
+free(q); \
+siridb_query_free(handle);
+
+static void QUERIES_free_merge_result(vec_t * plist);
+
+query_select_t * query_select_new(void)
+{
+ query_select_t * q_select = malloc(sizeof(query_select_t));
+
+ if (q_select == NULL)
+ {
+ return NULL;
+ }
+ QUERIES_NEW(q_select)
+
+ q_select->tp = QUERIES_SELECT;
+ q_select->start_ts = NULL;
+ q_select->end_ts = NULL;
+ q_select->presuf = NULL;
+ q_select->merge_as = NULL;
+ q_select->n = 0;
+ q_select->nselects = 1; /* we have at least one select function */
+ q_select->points_map = NULL;
+ q_select->alist = NULL;
+ q_select->mlist = NULL;
+ q_select->result = ct_new();
+
+ if (q_select->result == NULL)
+ {
+ free(q_select);
+ return NULL;
+ }
+
+ return q_select;
+}
+
+query_alter_t * query_alter_new(void)
+{
+ query_alter_t * q_alter = malloc(sizeof(query_alter_t));
+
+ if (q_alter == NULL)
+ {
+ return NULL;
+ }
+
+ QUERIES_NEW(q_alter)
+
+ q_alter->tp = QUERIES_ALTER;
+ q_alter->alter_tp = QUERY_ALTER_NONE;
+ q_alter->via.dummy = NULL;
+ q_alter->n = 0;
+
+ return q_alter;
+}
+
+query_count_t * query_count_new(void)
+{
+ query_count_t * q_count = malloc(sizeof(query_count_t));
+
+ if (q_count == NULL)
+ {
+ return NULL;
+ }
+
+ QUERIES_NEW(q_count)
+
+ q_count->tp = QUERIES_COUNT;
+ q_count->n = 0;
+
+ return q_count;
+}
+
+query_drop_t * query_drop_new(void)
+{
+ query_drop_t * q_drop = malloc(sizeof(query_drop_t));
+
+ if (q_drop == NULL)
+ {
+ return NULL;
+ }
+
+ QUERIES_NEW(q_drop)
+
+ q_drop->tp = QUERIES_DROP;
+ q_drop->n = 0;
+ q_drop->flags = 0;
+ q_drop->shards_list = NULL;
+
+ return q_drop;
+}
+
+query_list_t * query_list_new(void)
+{
+ query_list_t * q_list = malloc(sizeof(query_list_t));
+
+ if (q_list == NULL)
+ {
+ return NULL;
+ }
+
+ QUERIES_NEW(q_list)
+
+ q_list->tp = QUERIES_LIST;
+ q_list->props = NULL;
+ q_list->limit = DEFAULT_LIST_LIMIT;
+
+ return q_list;
+}
+
+void query_alter_free(uv_handle_t * handle)
+{
+ query_alter_t * q_alter = ((siridb_query_t *) handle->data)->data;
+
+ switch (q_alter->alter_tp)
+ {
+ case QUERY_ALTER_NONE:
+ case QUERY_ALTER_DATABASE:
+ case QUERY_ALTER_SERVERS:
+ case QUERY_ALTER_SERIES:
+ break;
+ case QUERY_ALTER_GROUP:
+ siridb_group_decref(q_alter->via.group);
+ break;
+ case QUERY_ALTER_TAG:
+ siridb_tag_decref(q_alter->via.tag);
+ break;
+ case QUERY_ALTER_SERVER:
+ siridb_server_decref(q_alter->via.server);
+ break;
+ case QUERY_ALTER_USER:
+ siridb_user_decref(q_alter->via.user);
+ break;
+ default:
+ assert(0);
+ }
+
+ QUERIES_FREE(q_alter, handle)
+}
+
+void query_count_free(uv_handle_t * handle)
+{
+ query_count_t * q_count = ((siridb_query_t *) handle->data)->data;
+
+ QUERIES_FREE(q_count, handle)
+}
+
+void query_drop_free(uv_handle_t * handle)
+{
+ query_drop_t * q_drop = ((siridb_query_t *) handle->data)->data;
+
+ if (q_drop->shards_list != NULL)
+ {
+ siridb_shard_t * shard;
+ while (q_drop->shards_list->len)
+ {
+ shard = vec_pop(q_drop->shards_list);
+ siridb_shard_decref(shard);
+ }
+
+ vec_free(q_drop->shards_list);
+ }
+
+ QUERIES_FREE(q_drop, handle)
+}
+
+void query_list_free(uv_handle_t * handle)
+{
+ query_list_t * q_list = ((siridb_query_t *) handle->data)->data;
+
+ if (q_list->props != NULL)
+ {
+ vec_free(q_list->props);
+ }
+
+ QUERIES_FREE(q_list, handle)
+}
+
+void query_select_free(uv_handle_t * handle)
+{
+ query_select_t * q_select = ((siridb_query_t *) handle->data)->data;
+
+ siridb_presuf_free(q_select->presuf);
+
+ if (q_select->points_map != NULL)
+ {
+ imap_free(q_select->points_map, (imap_free_cb) &siridb_points_free);
+ }
+
+ if (q_select->result != NULL)
+ {
+ if (q_select->merge_as == NULL)
+ {
+ ct_free(q_select->result, (ct_free_cb) &siridb_points_free);
+ }
+ else
+ {
+ ct_free(q_select->result, (ct_free_cb) &QUERIES_free_merge_result);
+ }
+ }
+
+ free(q_select->merge_as);
+
+ if (q_select->alist != NULL)
+ {
+ siridb_aggregate_list_free(q_select->alist);
+ }
+
+ if (q_select->mlist != NULL)
+ {
+ siridb_aggregate_list_free(q_select->mlist);
+ }
+
+ QUERIES_FREE(q_select, handle)
+}
+
+void query_help_free(uv_handle_t * handle)
+{
+ /* used as char to hold a string */
+ free(((siridb_query_t *) handle->data)->data);
+
+ /* normal call-back */
+ siridb_query_free(handle);
+}
+
+static void QUERIES_free_merge_result(vec_t * plist)
+{
+ size_t i;
+ for (i = 0; i < plist->len; i ++)
+ {
+ siridb_points_free(plist->data[i]);
+ }
+ free(plist);
+}
--- /dev/null
+/*
+ * query.c - Responsible for parsing queries.
+ */
+#include <assert.h>
+#include <cleri/cleri.h>
+#include <expr/expr.h>
+#include <iso8601/iso8601.h>
+#include <logger/logger.h>
+#include <siri/async.h>
+#include <siri/db/nodes.h>
+#include <siri/db/query.h>
+#include <siri/db/replicate.h>
+#include <siri/db/servers.h>
+#include <siri/db/time.h>
+#include <siri/db/walker.h>
+#include <siri/db/listener.h>
+#include <siri/db/queries.h>
+#include <siri/net/clserver.h>
+#include <siri/net/pkg.h>
+#include <siri/net/clserver.h>
+#include <siri/siri.h>
+#include <siri/grammar/gramp.h>
+#include <xstr/xstr.h>
+#include <string.h>
+#include <sys/time.h>
+#include <siri/err.h>
+
+#if SIRIDB_EXPR_ALLOC
+#include <llist/llist.h>
+#endif
+
+
+#define QUERY_TOO_LONG -1
+#define QUERY_MAX_LENGTH 8192
+#define QUERY_EXTRA_ALLOC_SIZE 200
+#define SIRIDB_FWD_SERVERS_TIMEOUT 5000 /* 5 seconds */
+
+static void QUERY_send_invalid_error(uv_async_t * handle);
+static void QUERY_parse(uv_async_t * handle);
+static int QUERY_walk(
+#if SIRIDB_EXPR_ALLOC
+ siridb_query_t * query,
+#endif
+ cleri_node_t * node,
+ siridb_walker_t * walker);
+static int QUERY_to_packer(qp_packer_t * packer, siridb_query_t * query);
+static int QUERY_time_expr(
+ cleri_node_t * node,
+ siridb_walker_t * walker,
+ char * buf,
+ size_t * size);
+static int QUERY_int_expr(cleri_node_t * node, char * buf, size_t * size);
+static int QUERY_rebuild(
+ siridb_t * siridb,
+ cleri_node_t * node,
+ char * buf,
+ size_t * size,
+ const size_t max_size);
+static void QUERY_send_no_query(uv_async_t * handle);
+
+/*
+ * This function can raise a SIGNAL.
+ */
+void siridb_query_run(
+ uint16_t pid,
+ sirinet_stream_t * client,
+ const char * q,
+ size_t q_len,
+ float factor,
+ int flags)
+{
+ uv_async_t * handle = malloc(sizeof(uv_async_t));
+ if (handle == NULL)
+ {
+ ERR_ALLOC
+ return;
+ }
+ siridb_query_t * query = malloc(sizeof(siridb_query_t));
+ if (query == NULL)
+ {
+ ERR_ALLOC
+ free(handle);
+ return;
+ }
+
+ /* set query */
+ if ((query->q = strndup(q, q_len)) == NULL)
+ {
+ ERR_ALLOC
+ free(query);
+ free(handle);
+ return;
+ }
+
+ #if SIRIDB_EXPR_ALLOC
+ if ((query->expr_cache = llist_new()) == NULL)
+ {
+ ERR_ALLOC
+ free(query->q);
+ free(query);
+ free(handle);
+ return;
+ }
+ #endif
+
+ /*
+ * Set start time.
+ * (must be real time since we translate now with this value)
+ */
+ clock_gettime(CLOCK_REALTIME, &query->start);
+
+ /* bind pid, client and flags so we can send back the result */
+ query->pid = pid;
+
+ /* increment client reference counter */
+ sirinet_stream_incref(client);
+
+ query->client = client;
+ query->flags = flags;
+
+ /* bind time precision factor */
+ query->factor = factor;
+
+ /* set the default callback, this might change when custom
+ * data is linked to the query handle
+ */
+ query->free_cb = siridb_query_free;
+ query->ref = 1;
+
+ /* We should initialize the packer based on query type */
+ query->packer = NULL;
+ query->timeit = NULL;
+
+ /* make sure all *other* pointers are set to NULL */
+ query->data = NULL;
+ query->pr = NULL;
+ query->nodes = NULL;
+
+ if (Logger.level == LOGGER_DEBUG && strstr(query->q, "password") == NULL)
+ {
+ log_debug("Parsing query (%d): %s", query->flags, query->q);
+ }
+
+ /* increment active tasks */
+ siridb_tasks_inc(client->siridb->tasks);
+
+ /* send next call */
+ uv_async_init(siri.loop, handle, (uv_async_cb) QUERY_parse);
+ handle->data = query;
+ uv_async_send(handle);
+}
+
+void siridb_query_free(uv_handle_t * handle)
+{
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ /* decrement active tasks */
+ siridb_tasks_dec(siridb->tasks);
+
+ /* free query */
+ free(query->q);
+
+ /* free qpack buffers */
+ if (query->packer != NULL)
+ {
+ qp_packer_free(query->packer);
+ }
+
+ if (query->timeit != NULL)
+ {
+ qp_packer_free(query->timeit);
+ }
+
+ /* free node list */
+ siridb_nodes_free(query->nodes);
+
+ /* free query result */
+ if (query->pr != NULL)
+ {
+ cleri_parse_free(query->pr);
+ }
+
+ #if SIRIDB_EXPR_ALLOC
+ if (query->expr_cache != NULL)
+ {
+ llist_destroy(query->expr_cache, (llist_destroy_cb) free);
+ }
+ #endif
+
+ /* decrement client reference counter */
+ sirinet_stream_decref(query->client);
+
+ /* free query */
+ free(query);
+
+ /* free handle */
+ free(handle);
+}
+
+void siridb_send_query_result(uv_async_t * handle)
+{
+ /*
+ * Its important to have another callback to free stuff so we can
+ * clean everything without sending things in case of a client failure
+ */
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+
+ assert (query->packer != NULL);
+
+ sirinet_pkg_t * pkg = sirinet_packer2pkg(
+ query->packer,
+ query->pid,
+ CPROTO_RES_QUERY);
+
+ sirinet_pkg_send(query->client, pkg);
+
+ query->packer = NULL;
+
+ uv_close((uv_handle_t *) handle, siri_async_close);
+}
+
+/*
+ * Signal can be raised by this function.
+ */
+void siridb_query_send_error(
+ uv_async_t * handle,
+ cproto_server_t err)
+{
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+
+ /* make sure the error message is null terminated in case the length has
+ * exceeded the max length.
+ */
+ query->err_msg[SIRIDB_MAX_SIZE_ERR_MSG-1] = '\0';
+
+ sirinet_pkg_t * package = sirinet_pkg_err(
+ query->pid,
+ strlen(query->err_msg),
+ err, /* usually this is CPROTO_ERR_QUERY, CPROTO_ERR_POOL...*/
+ query->err_msg);
+
+ log_warning("(%s) %s", sirinet_cproto_server_str(err), query->err_msg);
+
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(query->client, package);
+ }
+ uv_close((uv_handle_t *) handle, siri_async_close);
+}
+
+/*
+ * This function can raise a SIGNAL.
+ *
+ * Reference counter for handle will be incremented since we count the handle
+ * to a timer object. The cb function should perform siri_async_decref(&handle).
+ *
+ * (see siridb_query_fwd_t definition for info about fwd_t and flags)
+ */
+void siridb_query_forward(
+ uv_async_t * handle,
+ siridb_query_fwd_t fwd,
+ sirinet_promises_cb cb,
+ int flags)
+{
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ /*
+ * the size is important here, we will use the alloc_size to guess the
+ * maximum query size in QUERY_to_packer.
+ */
+ qp_packer_t * packer =
+ qp_packer_new(query->pr->tree->len + QUERY_EXTRA_ALLOC_SIZE);
+
+ if (packer == NULL)
+ {
+ ERR_ALLOC
+ return;
+ }
+
+ /*
+ * For backwards compatibility with SiriDB version < 2.0.24 we send an
+ * extra value SIRIDB_TIME_DEFAULT.
+ */
+ qp_add_type(packer, QP_ARRAY2);
+
+ /* add the query to the packer */
+ QUERY_to_packer(packer, query);
+ qp_add_int64(packer, SIRIDB_TIME_DEFAULT); /* Only for version < 2.0.24 */
+
+
+ sirinet_pkg_t * pkg = sirinet_pkg_new(0, packer->len, 0, packer->buffer);
+
+ /* increment reference since handle will be bound to a timer */
+ siri_async_incref(handle);
+
+ if (pkg != NULL)
+ {
+ switch (fwd)
+ {
+ case SIRIDB_QUERY_FWD_SERVERS:
+ pkg->tp = BPROTO_QUERY_SERVER;
+ {
+ vec_t * servers = siridb_servers_other2vec(siridb);
+ if (servers != NULL)
+ {
+ siridb_servers_send_pkg(
+ servers,
+ pkg,
+ SIRIDB_FWD_SERVERS_TIMEOUT,
+ cb,
+ handle);
+ vec_free(servers);
+ }
+ else
+ {
+ free(pkg);
+ ERR_ALLOC
+ }
+ }
+ break;
+
+ case SIRIDB_QUERY_FWD_POOLS:
+ pkg->tp = BPROTO_QUERY_SERVER;
+ siridb_pools_send_pkg(
+ siridb,
+ pkg,
+ 0,
+ cb,
+ handle,
+ flags);
+ break;
+
+ case SIRIDB_QUERY_FWD_SOME_POOLS:
+ assert (((query_select_t *) ((siridb_query_t *)
+ handle->data)->data)->tp == QUERIES_SELECT);
+ assert (((query_select_t *) ((siridb_query_t *)
+ handle->data)->data)->pmap != NULL);
+ pkg->tp = BPROTO_QUERY_SERVER;
+ {
+ vec_t * borrow_list = imap_vec(((query_select_t *) (
+ (siridb_query_t *) handle->data)->data)->pmap);
+ if (borrow_list != NULL)
+ {
+ /* if vec is NULL, a signal is raised */
+ siridb_pools_send_pkg_2some(
+ borrow_list,
+ pkg,
+ 0,
+ cb,
+ handle,
+ flags);
+ }
+ else
+ {
+ free(pkg);
+ }
+ }
+ break;
+
+ case SIRIDB_QUERY_FWD_UPDATE:
+ if (siridb->replica != NULL)
+ {
+ pkg->tp = BPROTO_QUERY_SERVER;
+ siridb_replicate_pkg(siridb, pkg);
+ }
+ pkg->tp = BPROTO_QUERY_UPDATE;
+ siridb_pools_send_pkg(
+ siridb,
+ pkg,
+ 0,
+ cb,
+ handle,
+ flags);
+ break;
+
+ default:
+ assert (0);
+ free(pkg);
+ break;
+ }
+ }
+ qp_packer_free(packer);
+}
+
+/*
+ * Unpack an error message from a package and copy the message
+ * to query->err_msg.
+ *
+ * Returns 0 if successful or -1 in case the package data is not valid.
+ */
+int siridb_query_err_from_pkg(siridb_query_t * query, sirinet_pkg_t * pkg)
+{
+ qp_unpacker_t unpacker;
+ qp_obj_t qp_err;
+
+ /* initialize unpacker */
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if (qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_is_raw(qp_next(&unpacker, NULL)) && /* error_msg */
+ qp_is_raw(qp_next(&unpacker, &qp_err)) &&
+ qp_err.len < SIRIDB_MAX_SIZE_ERR_MSG)
+ {
+ memcpy(query->err_msg, qp_err.via.raw, qp_err.len);
+ query->err_msg[qp_err.len] = '\0';
+ return 0;
+ }
+
+ /* package data did not have a valid error message */
+ return -1;
+}
+
+void siridb_query_timeit_from_unpacker(
+ siridb_query_t * query,
+ qp_unpacker_t * unpacker)
+{
+ assert (query->timeit != NULL);
+
+ qp_types_t tp = qp_next(unpacker, NULL);
+
+ while (qp_is_close(tp))
+ {
+ tp = qp_next(unpacker, NULL);
+ }
+
+ if ( qp_is_raw(tp) &&
+ qp_is_array(qp_next(unpacker, NULL)) &&
+ qp_is_map(qp_current(unpacker)))
+ {
+ qp_packer_extend_fu(query->timeit, unpacker);
+ }
+}
+
+static void QUERY_unique(cleri_olist_t * olist)
+{
+ while (olist != NULL && olist->next != NULL)
+ {
+ cleri_olist_t * test = olist;
+ while (test->next != NULL)
+ {
+ if (olist->cl_obj == test->next->cl_obj)
+ {
+ cleri_olist_t * tmp = test->next->next;
+ free(test->next);
+ test->next = tmp;
+ continue;
+ }
+ test = test->next;
+ }
+ olist = olist->next;
+ }
+}
+
+static void QUERY_send_invalid_error(uv_async_t * handle)
+{
+ size_t len;
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+ int count = 0;
+ const char * expect;
+ cleri_t * cl_obj;
+
+ /* start building the error message */
+ len = snprintf(query->err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Query error at position %zd. Expecting ",
+ query->pr->pos);
+
+ /* required for libcleri versions prior to 0.10.1 */
+ QUERY_unique(query->pr->expecting->required);
+
+ /* expand the error message with suggestions. we try to add nice names
+ * for regular expressions etc.
+ */
+ while (query->pr->expect != NULL)
+ {
+ cl_obj = query->pr->expect->cl_obj;
+ if (cl_obj->tp == CLERI_TP_END_OF_STATEMENT)
+ {
+ expect = "end_of_statement";
+ }
+ else if (cl_obj->tp == CLERI_TP_KEYWORD)
+ {
+ expect = cl_obj->via.keyword->keyword;
+ }
+ else if (cl_obj->tp == CLERI_TP_TOKENS)
+ {
+ expect = cl_obj->via.tokens->spaced;
+ }
+ else if (cl_obj->tp == CLERI_TP_TOKEN)
+ {
+ expect = cl_obj->via.token->token;
+ }
+ else switch (cl_obj->gid)
+ {
+ case CLERI_GID_R_SINGLEQ_STR:
+ expect = "single_quote_str"; break;
+ case CLERI_GID_R_DOUBLEQ_STR:
+ expect = "double_quote_str"; break;
+ case CLERI_GID_R_GRAVE_STR:
+ expect = "grave_str"; break;
+ case CLERI_GID_R_INTEGER:
+ expect = "integer"; break;
+ case CLERI_GID_R_FLOAT:
+ expect = "float"; break;
+ case CLERI_GID_R_UUID_STR:
+ expect = "uuid"; break;
+ case CLERI_GID_R_TIME_STR:
+ expect = "date/time_string"; break;
+ case CLERI_GID_R_REGEX:
+ expect = "regular_expression"; break;
+ default:
+ /* the best result we get is to handle all, but it will not break
+ * in case we did not specify some elements.
+ */
+ query->pr->expect = query->pr->expect->next;
+ continue;
+ }
+
+ /* make sure len is not greater than the maximum size */
+ if (len > SIRIDB_MAX_SIZE_ERR_MSG)
+ {
+ len = SIRIDB_MAX_SIZE_ERR_MSG;
+ }
+
+ /* we use count = 0 to print the first one, then for the others
+ * a comma prefix and the last with -or-
+ */
+ if (!count++)
+ {
+ len += snprintf(query->err_msg + len,
+ SIRIDB_MAX_SIZE_ERR_MSG - len,
+ "%s",
+ expect);
+ }
+ else if (query->pr->expect->next == NULL)
+ {
+ len += snprintf(query->err_msg + len,
+ SIRIDB_MAX_SIZE_ERR_MSG - len,
+ " or %s",
+ expect);
+ }
+ else
+ {
+ len += snprintf(query->err_msg + len,
+ SIRIDB_MAX_SIZE_ERR_MSG - len,
+ ", %s",
+ expect);
+ }
+
+ query->pr->expect = query->pr->expect->next;
+ }
+
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+}
+
+static void QUERY_send_no_query(uv_async_t * handle)
+{
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+
+ query->packer = sirinet_packer_new(512);
+ qp_add_type(query->packer, QP_MAP1);
+
+ siridb_t * siridb = query->client->siridb;
+
+ qp_add_raw(query->packer, (const unsigned char *) "calc", 4);
+ uint64_t ts = siridb_time_now(siridb, query->start);
+
+ if (!query->factor)
+ {
+ qp_add_int64(query->packer, (int64_t) ts);
+ }
+ else
+ {
+ double factor = (double) query->factor;
+ qp_add_int64(query->packer, (int64_t) (ts * factor));
+ }
+
+ siridb_send_query_result(handle);
+}
+
+static void QUERY_parse(uv_async_t * handle)
+{
+ int rc;
+ siridb_query_t * query = (siridb_query_t *) handle->data;
+ siridb_t * siridb = query->client->siridb;
+
+ siridb_walker_t * walker = siridb_walker_new(
+ siridb,
+ siridb_time_now(siridb, query->start),
+ &query->flags);
+
+ if ( walker == NULL ||
+ (query->pr = cleri_parse(siri.grammar, query->q)) == NULL)
+ {
+ if (walker != NULL)
+ {
+ siridb_nodes_free(siridb_walker_free(walker));
+ }
+ else
+ {
+ ERR_ALLOC
+ }
+ sprintf(query->err_msg,
+ "Memory allocation error or maximum recursion depth reached.");
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ if (!query->pr->is_valid)
+ {
+ siridb_nodes_free(siridb_walker_free(walker));
+ QUERY_send_invalid_error(handle);
+ return;
+ }
+
+ if ((rc = QUERY_walk(
+#if SIRIDB_EXPR_ALLOC
+ query,
+#endif
+ query->pr->tree->children->node,
+ walker)))
+ {
+ switch (rc)
+ {
+ case EXPR_DIVISION_BY_ZERO:
+ sprintf(query->err_msg, "Division by zero error.");
+ break;
+ case EXPR_MODULO_BY_ZERO:
+ sprintf(query->err_msg, "Modulo by zero error.");
+ break;
+ case EXPR_TOO_LONG:
+ sprintf(query->err_msg,
+ "Expression too long. (max %d characters)",
+ EXPR_MAX_SIZE);
+ break;
+ case EXPR_TIME_OUT_OF_RANGE:
+ sprintf(query->err_msg, "Time expression out-of-range.");
+ break;
+ case EXPR_INVALID_DATE_STRING:
+ sprintf(query->err_msg, "Invalid date string.");
+ break;
+ case EXPR_MEM_ALLOC_ERR:
+ sprintf(query->err_msg, "Memory allocation error.");
+ break;
+ default:
+ log_critical("Unknown Return Code received: %d", rc);
+ assert(0);
+ }
+ siridb_nodes_free(siridb_walker_free(walker));
+ siridb_query_send_error(handle, CPROTO_ERR_QUERY);
+ return;
+ }
+
+ /* free the walker but keep the nodes list */
+ query->nodes = siridb_walker_free(walker);
+
+ if (query->nodes == NULL)
+ {
+ QUERY_send_no_query(handle);
+ return;
+ }
+
+ uv_async_t * forward = malloc(sizeof(uv_async_t));
+ uv_async_init(siri.loop, forward, (uv_async_cb) query->nodes->cb);
+ forward->data = handle->data;
+ uv_async_send(forward);
+ uv_close((uv_handle_t *) handle, (uv_close_cb) free);
+}
+
+static int QUERY_to_packer(qp_packer_t * packer, siridb_query_t * query)
+{
+ int rc;
+ if (query->flags & SIRIDB_QUERY_FLAG_REBUILD)
+ {
+ /* reserve 200 extra chars */
+ char buffer[packer->alloc_size];
+ size_t size = packer->alloc_size;
+ siridb_t * siridb = query->client->siridb;
+
+ rc = QUERY_rebuild(
+ siridb,
+ query->pr->tree->children->node,
+ buffer,
+ &size,
+ packer->alloc_size);
+
+ if (rc)
+ {
+ log_error(
+ "Error '%d' occurred while re-building the query, "
+ "continue using the original query", rc);
+ qp_add_string(packer, query->q);
+ }
+ else
+ {
+ qp_add_raw(
+ packer,
+ (const unsigned char *) buffer,
+ packer->alloc_size - size);
+ }
+ }
+ else
+ {
+ qp_add_string(packer, query->q);
+ }
+ return 0;
+}
+
+static int QUERY_walk(
+#if SIRIDB_EXPR_ALLOC
+ siridb_query_t * query,
+#endif
+ cleri_node_t * node,
+ siridb_walker_t * walker)
+{
+ int rc;
+ uint32_t gid;
+ cleri_children_t * current;
+ uv_async_cb func;
+
+ gid = node->cl_obj->gid;
+
+ /*
+ * When GID is 0 this means CLERI_NONE
+ */
+ if (gid != CLERI_NONE)
+ {
+ if ( (func = siridb_node_get_enter(gid)) != NULL &&
+ siridb_walker_append(walker, node, func))
+ {
+ return EXPR_MEM_ALLOC_ERR;
+ }
+
+ if ( (func = siridb_node_get_exit(gid)) != NULL &&
+ siridb_walker_insert(walker, node, func))
+ {
+ return EXPR_MEM_ALLOC_ERR;
+ }
+ }
+
+ if (gid == CLERI_GID_TIME_EXPR || gid == CLERI_GID_CALC_STMT)
+ {
+ char buffer[EXPR_MAX_SIZE];
+ size_t size = EXPR_MAX_SIZE;
+
+ /* we can have nested integer and time expressions */
+ if ((rc = QUERY_time_expr(
+ node->children->node,
+ walker,
+ buffer,
+ &size)))
+ {
+ return rc;
+ }
+
+ /* terminate buffer */
+ buffer[EXPR_MAX_SIZE - size] = 0;
+
+ #if SIRIDB_EXPR_ALLOC
+ {
+ int64_t * itmp = malloc(sizeof(int64_t));
+ if (itmp == NULL || llist_append(query->expr_cache, itmp))
+ {
+ free(itmp);
+ return EXPR_MEM_ALLOC_ERR;
+ }
+ node->data = itmp;
+ }
+ #endif
+
+ /* evaluate the expression */
+ if ((rc = expr_parse(CLERI_NODE_DATA_ADDR(node), buffer)))
+ {
+ return rc;
+ }
+
+ /* check if timestamp is valid */
+ if (!siridb_int64_valid_ts(walker->siridb->time, CLERI_NODE_DATA(node)))
+ {
+ return EXPR_TIME_OUT_OF_RANGE;
+ }
+ }
+ else if (gid == CLERI_GID_INT_EXPR)
+ {
+ char buffer[EXPR_MAX_SIZE];
+ size_t size = EXPR_MAX_SIZE;
+
+ if ((rc = QUERY_int_expr(
+ node->children->node,
+ buffer,
+ &size)))
+ {
+ return rc;
+ }
+
+ /* terminate buffer */
+ buffer[EXPR_MAX_SIZE - size] = 0;
+
+ #if SIRIDB_EXPR_ALLOC
+ {
+ int64_t * itmp = malloc(sizeof(int64_t));
+ if (itmp == NULL || llist_append(query->expr_cache, itmp))
+ {
+ free(itmp);
+ return EXPR_MEM_ALLOC_ERR;
+ }
+ node->data = itmp;
+ }
+ #endif
+
+ /* evaluate the expression */
+ if ((rc = expr_parse(CLERI_NODE_DATA_ADDR(node), buffer)))
+ {
+ return rc;
+ }
+ }
+ else
+ {
+ current = node->children;
+ while (current != NULL && current->node != NULL)
+ {
+ /*
+ * We should not simple walk because THIS has no
+ * cl_obj->cl_obj and THIS is save to skip.
+ */
+ while (current->node->cl_obj->tp == CLERI_TP_THIS)
+ {
+ current = current->node->children;
+ }
+ if ((rc = QUERY_walk(
+#if SIRIDB_EXPR_ALLOC
+ query,
+#endif
+ current->node,
+ walker)))
+ {
+ return rc;
+ }
+ current = current->next;
+ }
+ }
+
+ return 0;
+}
+
+static int QUERY_time_expr(
+ cleri_node_t * node,
+ siridb_walker_t * walker,
+ char * buf,
+ size_t * size)
+{
+ int n;
+
+ switch (node->cl_obj->tp)
+ {
+ case CLERI_TP_TOKEN:
+ case CLERI_TP_TOKENS:
+ /* tokens like + - ( etc. */
+ *(buf + EXPR_MAX_SIZE - *size) = *node->str;
+ return (--(*size)) ? 0 : EXPR_TOO_LONG;
+
+ case CLERI_TP_KEYWORD:
+ /* this is now */
+ n = snprintf(
+ buf + EXPR_MAX_SIZE - *size,
+ *size,
+ "%" PRIu64,
+ walker->now);
+ if (n >= (ssize_t) *size)
+ {
+ return EXPR_TOO_LONG;
+ }
+ *walker->flags |= SIRIDB_QUERY_FLAG_REBUILD;
+ *size -= n;
+ return 0;
+
+ case CLERI_TP_REGEX:
+ /* can be an integer or time string like 2d or something */
+ switch (node->cl_obj->gid)
+ {
+ case CLERI_GID_R_INTEGER:
+ if (node->len >= *size)
+ {
+ return EXPR_TOO_LONG;
+ }
+ memcpy(buf + EXPR_MAX_SIZE - *size, node->str, node->len);
+ *size -= node->len;
+ return 0;
+
+ case CLERI_GID_R_TIME_STR:
+ n = snprintf(
+ buf + EXPR_MAX_SIZE - *size,
+ *size,
+ "%" PRIu64,
+ siridb_time_parse(node->str, node->len) *
+ walker->siridb->time->factor);
+ if (n >= (ssize_t) *size)
+ {
+ return EXPR_TOO_LONG;
+ }
+ *size -= n;
+ return 0;
+ }
+ log_critical(
+ "Unexpected object in time expression received: %d",
+ node->cl_obj->tp);
+ assert (0);
+ break;
+
+ case CLERI_TP_CHOICE:
+ /* this is a string (single or double quoted) */
+ {
+ char datestr[node->len - 1];
+
+ /* extract date string */
+ xstr_extract_string(datestr, node->str, node->len);
+
+ /* get timestamp from date string */
+ int64_t ts = iso8601_parse_date(datestr, walker->siridb->tz);
+
+ if (ts < 0)
+ {
+ return EXPR_INVALID_DATE_STRING;
+ }
+
+ n = snprintf(
+ buf + EXPR_MAX_SIZE - *size,
+ *size,
+ "%" PRId64,
+ ts * walker->siridb->time->factor);
+
+ if (n >= (ssize_t) *size)
+ {
+ return EXPR_TOO_LONG;
+ }
+ *walker->flags |= SIRIDB_QUERY_FLAG_REBUILD;
+ *size -= n;
+ }
+ return 0;
+
+ default:
+ /* anything else, probably THIS or a sequence */
+ {
+ int rc;
+ cleri_children_t * current;
+
+ current = node->children;
+ while (current != NULL && current->node != NULL)
+ {
+ if ((rc = QUERY_time_expr(
+ current->node,
+ walker,
+ buf,
+ size)))
+ {
+ return rc;
+ }
+ current = current->next;
+ }
+ }
+ }
+ return 0;
+}
+
+static int QUERY_int_expr(cleri_node_t * node, char * buf, size_t * size)
+{
+ switch (node->cl_obj->tp)
+ {
+ case CLERI_TP_TOKEN:
+ case CLERI_TP_TOKENS:
+ /* tokens like + - ( etc. */
+ *(buf + EXPR_MAX_SIZE - *size) = *node->str;
+ return (--(*size)) ? 0 : EXPR_TOO_LONG;
+
+ case CLERI_TP_REGEX:
+ /* this is an integer */
+ if (node->len >= *size)
+ {
+ return EXPR_TOO_LONG;
+ }
+ memcpy(buf + EXPR_MAX_SIZE - *size, node->str, node->len);
+ *size -= node->len;
+ return 0;
+
+ default:
+ /* anything else, probably THIS or a sequence */
+ {
+ int rc;
+ cleri_children_t * current;
+
+ current = node->children;
+ while (current != NULL && current->node != NULL)
+ {
+ if ((rc = QUERY_int_expr(
+ current->node,
+ buf,
+ size)))
+ {
+ return rc;
+ }
+ current = current->next;
+ }
+ }
+ }
+ return 0;
+}
+
+/*
+ * Returns 0 if successful or QUERY_TOO_LONG
+ */
+static int QUERY_rebuild(
+ siridb_t * siridb,
+ cleri_node_t * node,
+ char * buf,
+ size_t * size,
+ const size_t max_size)
+{
+ switch (node->cl_obj->tp)
+ {
+ case CLERI_TP_REGEX:
+ case CLERI_TP_TOKEN:
+ case CLERI_TP_TOKENS:
+ case CLERI_TP_KEYWORD:
+ if (node->len >= *size)
+ {
+ return QUERY_TOO_LONG;
+ }
+ memcpy(buf + max_size - *size, node->str, node->len);
+ *size -= node->len;
+
+ *(buf + max_size - *size) = ' ';
+ return (--(*size)) ? 0 : QUERY_TOO_LONG;
+
+ case CLERI_TP_CHOICE:
+ case CLERI_TP_RULE:
+ switch (node->cl_obj->gid)
+ {
+ case CLERI_GID_UUID:
+ {
+ siridb_server_t * server = siridb_server_from_node(
+ siridb,
+ node->children->node,
+ NULL);
+ if (server != NULL)
+ {
+ char uuid[37];
+ uuid_unparse_lower(server->uuid, uuid);
+ int n;
+ n = snprintf(
+ buf + max_size - *size,
+ *size,
+ "%s ",
+ uuid);
+ if (n >= (ssize_t) *size)
+ {
+ return QUERY_TOO_LONG;
+ }
+ *size -= n;
+ return 0;
+ }
+ }
+ break;
+ case CLERI_GID_INT_EXPR:
+ case CLERI_GID_TIME_EXPR:
+ {
+ int n;
+ n = snprintf(
+ buf + max_size - *size,
+ *size,
+ "%" PRId64 " ",
+ CLERI_NODE_DATA(node));
+ if (n >= (ssize_t) *size)
+ {
+ return QUERY_TOO_LONG;
+ }
+ *size -= n;
+ }
+ return 0;
+ }
+ /* FALLTHRU */
+ /* fall through */
+ default:
+ {
+ int rc;
+ cleri_children_t * current;
+
+ current = node->children;
+ while (current != NULL && current->node != NULL)
+ {
+ if ((rc = QUERY_rebuild(
+ siridb,
+ current->node,
+ buf,
+ size,
+ max_size)))
+ {
+ return rc;
+ }
+ current = current->next;
+ }
+ }
+ }
+ return 0;
+}
--- /dev/null
+/*
+ * re.c - Helpers for regular expressions.
+ */
+#include <assert.h>
+#include <siri/db/db.h>
+#include <siri/db/re.h>
+
+/*
+ * Compiles both a 'pcre' regular expression and 'pcre_extra' optimization if
+ * the expression could be optimized.
+ *
+ * When successful, 0 is returned. In case of an error, -1 is returned and
+ * the 'err_msg' is set to an appropriate error message. Both 'regex' and
+ * 'regex_extra' are NULL when an error is returned.
+ *
+ * (SIRIDB_MAX_SIZE_ERR_MSG is honored for the error message)
+ */
+int siridb_re_compile(
+ pcre2_code ** regex,
+ pcre2_match_data ** match_data,
+ const char * source,
+ size_t len,
+ char * err_msg)
+{
+ int options = 0;
+ int pcre_error_num;
+ PCRE2_SIZE pcre_error_offset;
+ char pattern[len + 1];
+ memcpy(pattern, source, len);
+ pattern[0] = '^';
+
+ switch (pattern[--len])
+ {
+ case 'i':
+ options |= PCRE2_CASELESS;
+ len--;
+ /* FALLTHRU */
+ /* fall through */
+ case '/':
+ pattern[len] = '$';
+ pattern[len + 1] = '\0';
+ break;
+ default:
+ /* by the grammar definition, only the optional 'i' is allowed */
+ assert (0);
+ break;
+ }
+
+ *regex = pcre2_compile(
+ (PCRE2_SPTR8) pattern,
+ PCRE2_ZERO_TERMINATED,
+ options,
+ &pcre_error_num,
+ &pcre_error_offset,
+ NULL);
+
+ if (*regex == NULL)
+ {
+ PCRE2_UCHAR buffer[256];
+ pcre2_get_error_message(pcre_error_num, buffer, sizeof(buffer));
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot compile regular expression '%s': %s",
+ pattern,
+ buffer);
+
+ return -1;
+ }
+
+ *match_data = pcre2_match_data_create_from_pattern(*regex, NULL);
+
+ /*
+ * pcre_study() returns NULL for both errors and when it can not
+ * optimize the regex. The last argument is how one checks for
+ * errors (it is NULL if everything works, and points to an error
+ * string otherwise.
+ */
+ if(*match_data == NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot create match data for regular expression '%s'",
+ pattern);
+
+ /* free and set regex back to NULL */
+ pcre2_code_free(*regex);
+ *regex = NULL;
+
+ return -1;
+ }
+
+ return 0;
+}
--- /dev/null
+/*
+ * reindex.c - SiriDB Re-index.
+ *
+ * Differences while re-indexing:
+ *
+ * - Group information like number of series will be updated at a lower
+ * interval which leads to probably incorrect number of series per group.
+ * Selections for series in a group or a list of series per group are still
+ * correct and can only lack of brand new series. (newer than 30 seconds)
+ *
+ * - Selecting an unknown series usually raises a QueryError but we do not
+ * raise this error during re-indexing since the series might be in either
+ * the old- or new pool. (selecting series during re-indexing has therefore
+ * the same behavior as a regular expression selection)
+ *
+ * - Drop server is not allowed while re-indexing.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <logger/logger.h>
+#include <qpack/qpack.h>
+#include <siri/db/db.h>
+#include <siri/db/reindex.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/err.h>
+#include <siri/net/protocol.h>
+#include <siri/optimize.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#define REINDEX_SLEEP 100 /* 100 milliseconds * active tasks */
+#define REINDEX_RETRY 5000 /* 5 seconds */
+#define REINDEX_INITWAIT 20000 /* 20 seconds */
+#define REINDEX_TIMEOUT 300000 /* 5 minutes */
+
+#define NEXT_SERIES_ERR -1
+#define NEXT_SERIES_SET 0
+#define NEXT_SERIES_END 1
+
+static const long int SIZE2 = 2 * sizeof(uint32_t);
+static const size_t PCKSZ = sizeof(sirinet_pkg_t) + 5;
+
+static inline int REINDEX_fn(siridb_t * siridb, siridb_reindex_t * reindex);
+static int REINDEX_create_cb(siridb_series_t * series, FILE * fp);
+static int REINDEX_unlink(siridb_reindex_t * reindex);
+static int REINDEX_next_series_id(siridb_reindex_t * reindex);
+static void REINDEX_next(siridb_t * siridb);
+static void REINDEX_work(uv_timer_t * timer);
+static void REINDEX_commit_series(siridb_t * siridb);
+static void REINDEX_on_insert_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+static void REINDEX_on_tag_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+
+static char reindex_progress[30];
+
+/*
+ * Returns a pointer to reindex. If 'create_new' is zero and an
+ * re-index file cannot be found, NULL is returned.
+ *
+ * In case of an error we also return NULL but in this case a SIGNAL is raised
+ */
+siridb_reindex_t * siridb_reindex_open(siridb_t * siridb, int create_new)
+{
+ siridb_reindex_t * reindex = malloc(sizeof(siridb_reindex_t));
+ if (reindex == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ reindex->fn = NULL;
+ reindex->fp = NULL;
+ reindex->next_series_id = NULL;
+ reindex->pkg_points = NULL;
+ reindex->pkg_tags = NULL;
+ reindex->timer = NULL;
+ reindex->server = NULL;
+ if (REINDEX_fn(siridb, reindex) < 0)
+ {
+ ERR_ALLOC
+ siridb_reindex_free(&reindex);
+ }
+ else
+ {
+ siridb_reindex_fopen(reindex, create_new ? "w+" : "r+");
+ if (reindex->fp == NULL)
+ {
+ siridb_reindex_free(&reindex);
+ }
+ else
+ {
+ siridb->flags |= SIRIDB_FLAG_REINDEXING;
+
+ if (create_new)
+ {
+ if (imap_walk(
+ siridb->series_map,
+ (imap_cb) REINDEX_create_cb,
+ reindex->fp) || fflush(reindex->fp))
+ {
+ ERR_FILE
+ siridb_reindex_free(&reindex);
+ }
+ }
+ else
+ {
+ siridb->pools->prev_lookup =
+ siridb_lookup_new(siridb->pools->len - 1);
+ if (siridb->pools->prev_lookup == NULL)
+ {
+ siridb_reindex_free(&reindex); /* signal is raised */
+ }
+ }
+
+ if (reindex != NULL)
+ {
+ reindex->size = ftello(reindex->fp);
+ if (reindex->size == -1)
+ {
+ ERR_FILE
+ siridb_reindex_free(&reindex);
+ }
+ else if (reindex->size)
+ {
+ reindex->next_series_id = malloc(sizeof(uint32_t));
+
+ if (reindex->next_series_id == NULL)
+ {
+ ERR_ALLOC
+ siridb_reindex_free(&reindex);
+ }
+ else if (
+ fseeko(reindex->fp, -sizeof(uint32_t), SEEK_END) ||
+ fread( reindex->next_series_id,
+ sizeof(uint32_t),
+ 1,
+ reindex->fp) != 1)
+ {
+ ERR_FILE
+ siridb_reindex_free(&reindex);
+ }
+ else
+ {
+ reindex->timer = malloc(sizeof(uv_timer_t));
+ if (reindex->timer == NULL)
+ {
+ ERR_ALLOC
+ siridb_reindex_free(&reindex);
+ }
+ else
+ {
+ reindex->server = siridb->pools->pool[
+ siridb->pools->len -1].server[0];
+ siridb->server->flags |= SERVER_FLAG_REINDEXING;
+ reindex->timer->data = siridb;
+ siri_optimize_pause();
+
+ uv_timer_init(siri.loop, reindex->timer);
+ if (create_new)
+ {
+ /*
+ * Sending the flags is only needed when
+ * the re-index was just created. Otherwise
+ * the flags are send when we authenticate.
+ */
+ siridb_servers_send_flags(siridb->servers);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return reindex;
+}
+
+/*
+ * Returns a human readable re-index progress status
+ */
+const char * siridb_reindex_progress(siridb_t * siridb)
+{
+ if ( siridb->reindex == NULL ||
+ siridb->reindex->timer == NULL ||
+ !siridb->reindex->size)
+ {
+ sprintf(reindex_progress, "not available");
+ }
+ else
+ {
+ size_t num = siridb->reindex->size / sizeof(uint32_t);
+ size_t total = siridb->series_map->len;
+ double percent = 100 * (double) (total - num) / total;
+
+ sprintf(reindex_progress,
+ "approximately at %0.2f%%",
+ (0 > percent) ? 0 : percent);
+ }
+ return reindex_progress;
+}
+
+/*
+ * This will close the re-index task and sets 'reindex->timer' to NULL.
+ *
+ * Warning: make sure open promises are closed since the 'REINDEX_work'
+ * depends on having siridb->reindex.
+ */
+void siridb_reindex_close(siridb_reindex_t * reindex)
+{
+ assert (reindex != NULL && reindex->timer != NULL);
+
+ /* we can use uv_timer_stop() even if the timer is not scheduled */
+ uv_timer_stop(reindex->timer);
+ uv_close((uv_handle_t *) reindex->timer, (uv_close_cb) free);
+ reindex->timer = NULL;
+}
+
+/*
+ * Destroy *reindex and set reindex to NULL. Parsing *reindex == NULL is
+ * not allowed and reindex->timer must be NULL. (use 'siridb_reindex_close' to
+ * close and set the timer to NULL)
+ *
+ * (a SIGNAL is raised in case the file cannot be closed)
+ */
+void siridb_reindex_free(siridb_reindex_t ** reindex)
+{
+ assert ((*reindex)->timer == NULL);
+ if ((*reindex)->fp != NULL && fclose((*reindex)->fp))
+ {
+ ERR_FILE
+ }
+ free((*reindex)->fn);
+ free((*reindex)->next_series_id);
+ free((*reindex)->pkg_points);
+ free((*reindex)->pkg_tags);
+ free(*reindex);
+ *reindex = NULL;
+}
+
+/*
+ * Open the reindex file. (set both the file pointer and file descriptor
+ * In case of and error, reindex->fp is set to NULL
+ */
+void siridb_reindex_fopen(siridb_reindex_t * reindex, const char * opentype)
+{
+ reindex->fp = fopen(reindex->fn, opentype);
+ if (reindex->fp != NULL)
+ {
+ reindex->fd = fileno(reindex->fp);
+ if (reindex->fd == -1)
+ {
+ log_critical("Error reading file descriptor: '%s'", reindex->fn);
+ fclose(reindex->fp);
+ reindex->fp = NULL;
+ }
+ }
+}
+
+/*
+ * Only call this function when the re-index flag is set and the server
+ * re-index flag is not set.
+ *
+ * This task can destroy siridb->reindex when all servers are finished with
+ * re-indexing.
+ */
+void siridb_reindex_status_update(siridb_t * siridb)
+{
+ assert (~siridb->server->flags & SERVER_FLAG_REINDEXING);
+ assert (siridb->flags & SIRIDB_FLAG_REINDEXING);
+ if (siridb_servers_available(siridb))
+ {
+ siridb->flags &= ~SIRIDB_FLAG_REINDEXING;
+
+ /* free previous lookup */
+ siridb_lookup_free(siridb->pools->prev_lookup);
+ siridb->pools->prev_lookup = NULL;
+
+ REINDEX_unlink(siridb->reindex);
+ siridb_reindex_free(&siridb->reindex);
+ log_info("Finished re-indexing database '%s'", siridb->dbname);
+ }
+}
+
+/*
+ * Make sure the optimize task is set to paused before calling this function.
+ * Its not required that the optimize task is already in PAUSED status.
+ * In that case we just wait for the PAUSED status.
+ */
+void siridb_reindex_start(uv_timer_t * timer)
+{
+ if (timer == NULL)
+ {
+ log_debug("No series are scheduled for re-indexing");
+ }
+ else
+ {
+ assert (siri.optimize->pause);
+ if (!SIRI_OPTIMZE_IS_PAUSED)
+ {
+ log_debug("Wait for the optimize task to pause");
+ uv_timer_start(timer, siridb_reindex_start, 1000, 0);
+ }
+ else
+ {
+ log_debug("Start re-indexing task in %u seconds",
+ REINDEX_INITWAIT / 1000);
+ uv_timer_start(timer, REINDEX_work, REINDEX_INITWAIT, 0);
+ }
+ }
+}
+
+/*
+ * Type: uv_timer_cb
+ *
+ * This function sends a packed series to the new (pool) server.
+ */
+static void REINDEX_send(uv_timer_t * timer)
+{
+ siridb_t * siridb = (siridb_t *) timer->data;
+ assert (siridb->reindex->pkg_points != NULL);
+ /* actually 'available' is sufficient since the destination server has
+ * never status 're-indexing' unless one day we support down-scaling.
+ */
+ if (siridb_server_is_accessible(siridb->reindex->server))
+ {
+ siridb_server_send_pkg(
+ siridb->reindex->server,
+ siridb->reindex->pkg_points,
+ REINDEX_TIMEOUT,
+ (sirinet_promise_cb) REINDEX_on_insert_response,
+ siridb,
+ FLAG_KEEP_PKG);
+ }
+ else
+ {
+ log_info("Cannot send re-index package to '%s' "
+ "(try again in %d seconds)",
+ siridb->reindex->server->name,
+ REINDEX_RETRY / 1000);
+ uv_timer_start(
+ siridb->reindex->timer,
+ REINDEX_send,
+ REINDEX_RETRY,
+ 0);
+ }
+}
+
+/*
+ * Return values:
+ * NEXT_SERIES_SET: Successful set the next series id and removes the previous
+ * series id
+ * NEXT_SERIES_END: End of the file is reached, re-indexing has finished.
+ * NEXT_SERIES_ERR: An error occurred and a SIGNAL is raised
+ */
+static int REINDEX_next_series_id(siridb_reindex_t * reindex)
+{
+ /* free re-index package */
+ free(reindex->pkg_points);
+ free(reindex->pkg_tags);
+ reindex->pkg_points = NULL;
+ reindex->pkg_tags = NULL;
+
+ int rc;
+ reindex->size -= sizeof(uint32_t);
+
+ if (!reindex->size)
+ {
+ rc = NEXT_SERIES_END;
+ }
+ else
+ {
+ if (fseeko(reindex->fp, -SIZE2, SEEK_END) ||
+ fread( reindex->next_series_id,
+ sizeof(uint32_t),
+ 1,
+ reindex->fp) != 1)
+ {
+ ERR_FILE
+ log_critical("Reading next series id has failed");
+ return NEXT_SERIES_ERR;
+ }
+ rc = NEXT_SERIES_SET;
+ }
+
+ if (ftruncate(reindex->fd, reindex->size))
+ {
+ ERR_FILE
+ log_critical("Reading next series id has failed");
+ return NEXT_SERIES_ERR;
+ }
+ return rc;
+}
+
+/*
+ * Call-back function: sirinet_promise_cb
+ */
+static void REINDEX_on_empty_tags_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ if (status)
+ {
+ log_error("Error while sending empty tags (%d)", status);
+ }
+ else if (sirinet_protocol_is_error(pkg->tp))
+ {
+ log_error(
+ "Error occurred while processing data on the new server: "
+ "(response type: %u)", pkg->tp);
+ }
+
+ sirinet_promise_decref(promise);
+}
+
+/*
+ * This function can raise a SIGNAL
+ */
+static void REINDEX_next(siridb_t * siridb)
+{
+ switch (REINDEX_next_series_id(siridb->reindex))
+ {
+ case NEXT_SERIES_SET:
+ uv_timer_start(
+ siridb->reindex->timer,
+ REINDEX_work,
+ REINDEX_SLEEP * siridb->tasks.active,
+ 0);
+ break;
+
+ case NEXT_SERIES_END:
+ {
+ sirinet_pkg_t * pkg;
+
+ /* send empty tags if required */
+ pkg = siridb_tags_empty(siridb->tags);
+ if (pkg)
+ {
+ if (siridb_server_send_pkg(
+ siridb->reindex->server,
+ pkg,
+ REINDEX_TIMEOUT,
+ (sirinet_promise_cb) REINDEX_on_empty_tags_response,
+ NULL,
+ 0))
+ {
+ free(pkg);
+ }
+ }
+
+ /* update and send the flags */
+ siridb->server->flags &= ~SERVER_FLAG_REINDEXING;
+ siridb_servers_send_flags(siridb->servers);
+
+ log_info("Re-indexing has successfully finished on '%s'",
+ siridb->server->name);
+
+ /* we can close the timer */
+ siridb_reindex_close(siridb->reindex);
+
+ /* check if everyone is finished and if so destroy re-index */
+ siridb_reindex_status_update(siridb);
+
+ siri_optimize_continue();
+ break;
+ }
+ case NEXT_SERIES_ERR:
+ break; /* signal is raised */
+ }
+}
+
+static void REINDEX_work(uv_timer_t * timer)
+{
+ siridb_t * siridb = (siridb_t *) timer->data;
+ siridb_reindex_t * reindex = siridb->reindex;
+
+ assert (SIRI_OPTIMZE_IS_PAUSED);
+ assert (reindex != NULL);
+ assert (siridb->reindex->pkg_points == NULL);
+ assert (siridb->reindex->pkg_tags == NULL);
+
+ reindex->series = imap_get(siridb->series_map, *reindex->next_series_id);
+
+ if ( reindex->series == NULL ||
+ siridb_lookup_sn(
+ siridb->pools->lookup,
+ reindex->series->name) == siridb->server->pool ||
+ (siridb->replica != NULL &&
+ siridb_series_server_id(reindex->series) != siridb->server->id))
+ {
+ REINDEX_next(siridb);
+ }
+ else
+ {
+ /*
+ * lock is not needed since we are sure the optimize task is
+ * not running
+ */
+ assert (siridb_lookup_sn(
+ siridb->pools->prev_lookup,
+ reindex->series->name) == siridb->server->pool);
+ siridb_points_t * points = siridb_series_get_points(
+ reindex->series,
+ NULL,
+ NULL);
+
+ if (points != NULL) /* signal is raised in case NULL */
+ {
+ /* tag package may be NULL when no tag need to be
+ * synchronized */
+ reindex->pkg_tags = siridb_tags_series(reindex->series);
+
+ /*
+ * Prepare drop, increasing the reference counter is not needed
+ * since the series can only be decremented when dropped. since
+ * the series is not member of the siridb->series_map it will not
+ * be decremented there either.
+ */
+
+ siridb_series_drop_prepare(siridb, reindex->series);
+
+ qp_packer_t * packer = sirinet_packer_new(QP_SUGGESTED_SIZE);
+ if (packer != NULL)
+ {
+ qp_add_type(packer, QP_MAP1);
+
+ /* add series name including terminator char */
+ qp_add_raw(
+ packer,
+ (const unsigned char *) reindex->series->name,
+ reindex->series->name_len + 1);
+
+ if (siridb_points_pack(points, packer) == 0)
+ {
+ reindex->pkg_points = sirinet_packer2pkg(
+ packer,
+ 0,
+ BPROTO_INSERT_TESTED_SERVER);
+
+ uv_timer_start(
+ reindex->timer,
+ REINDEX_send,
+ 0,
+ 0);
+ }
+ else
+ {
+ qp_packer_free(packer); /* signal raised */
+ }
+ }
+ siridb_points_free(points);
+ }
+ }
+}
+
+/*
+ * Sends 'dropped series' to the replica and commit the drop.
+ *
+ * This function can raise an ALLOC error but file errors are only logged.
+ */
+static void REINDEX_commit_series(siridb_t * siridb)
+{
+ /*
+ * Send the dropped series to the replica. The replica server might have
+ * received points between the time we prepared the drop and commit the
+ * drop but this is not an issue because the replica then forwards the
+ * points to this server and this server to the new server since the series
+ * is not in 'this' series store anymore
+ */
+ if (siridb->replica != NULL)
+ {
+ size_t len = siridb->reindex->series->name_len + 1;
+ qp_packer_t * packer = sirinet_packer_new(PCKSZ + len);
+ if (packer != NULL)
+ {
+ /* no need for testing, fits for sure */
+ qp_add_raw(
+ packer,
+ (const unsigned char *) siridb->reindex->series->name,
+ len);
+ sirinet_pkg_t * pkg = sirinet_packer2pkg(
+ packer,
+ 0,
+ BPROTO_DROP_SERIES);
+ siridb_replicate_pkg(siridb, pkg);
+ free(pkg);
+ }
+ }
+
+ if (siridb->reindex->pkg_tags)
+ {
+ siridb_server_send_pkg(
+ siridb->reindex->server,
+ siridb->reindex->pkg_tags,
+ REINDEX_TIMEOUT,
+ (sirinet_promise_cb) REINDEX_on_tag_response,
+ NULL,
+ FLAG_KEEP_PKG);
+ }
+
+ /* commit the drop */
+ if (siridb_series_drop_commit(siridb, siridb->reindex->series) == 0)
+ {
+ siridb_series_flush_dropped(siridb);
+ }
+}
+
+/*
+ * Call-back function: sirinet_promise_cb
+ */
+static void REINDEX_on_insert_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ siridb_t * siridb = (siridb_t *) promise->data;
+
+ switch ((sirinet_promise_status_t) status)
+ {
+ case PROMISE_WRITE_ERROR:
+ /*
+ * Write to socket error, data is not send so we should not commit.
+ */
+ uv_timer_start(
+ siridb->reindex->timer,
+ REINDEX_send,
+ REINDEX_RETRY,
+ 0);
+ break;
+ case PROMISE_TIMEOUT_ERROR:
+ /*
+ * Timeout occurred, use commit error since the data can be
+ * processed by the replica, we're just not sure.
+ */
+ case PROMISE_CANCELLED_ERROR:
+ /*
+ * Promise is cancelled but most likely the data is successful
+ * processed. Use siridb_fifo_commit_err() since we're not sure.
+ */
+ case PROMISE_PKG_TYPE_ERROR:
+ /*
+ * Commit with error since this package has result in an unknown
+ * package type.
+ */
+ log_error("Error occurred while sending series to the new server (%d)",
+ status);
+ REINDEX_commit_series(siridb);
+ REINDEX_next(siridb);
+ break;
+ case PROMISE_SUCCESS:
+ if (sirinet_protocol_is_error(pkg->tp))
+ {
+ log_error(
+ "Error occurred while processing data on the new server: "
+ "(response type: %u)", pkg->tp);
+ }
+ REINDEX_commit_series(siridb);
+ REINDEX_next(siridb);
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ sirinet_promise_decref(promise);
+}
+
+/*
+ * Call-back function: sirinet_promise_cb
+ */
+static void REINDEX_on_tag_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ if (status)
+ {
+ log_error("Error while sending tags (%d)", status);
+ }
+ else if (sirinet_protocol_is_error(pkg->tp))
+ {
+ log_error(
+ "Error occurred while processing data on the new server: "
+ "(response type: %u)", pkg->tp);
+ }
+
+ sirinet_promise_decref(promise);
+}
+
+/*
+ * Typedef: imap_cb
+ *
+ * Returns 0 if successful
+ */
+static int REINDEX_create_cb(siridb_series_t * series, FILE * fp)
+{
+ return fwrite(&series->id, sizeof(uint32_t), 1, fp) - 1;
+}
+
+/*
+ * Set reindex->fn to the correct file name.
+ *
+ * Returns the length of 'fn' or a negative value in case of an error.
+ */
+static inline int REINDEX_fn(siridb_t * siridb, siridb_reindex_t * reindex)
+{
+ return asprintf(
+ &reindex->fn,
+ "%s%s",
+ siridb->dbpath,
+ REINDEX_FN);
+}
+
+/*
+ * Close the file pointer and remove the re-index file.
+ * Return 0 if successful or -1 in case of an error.
+ */
+static int REINDEX_unlink(siridb_reindex_t * reindex)
+{
+ assert (reindex->fp != NULL);
+ fclose(reindex->fp);
+ reindex->fp = NULL;
+
+ if (unlink(reindex->fn))
+ {
+ log_critical(
+ "Re-index file could not be removed: '%s'",
+ reindex->fn);
+ return -1;
+ }
+ else
+ {
+ log_info("Re-index file removed: '%s'", reindex->fn);
+ }
+ return 0;
+}
--- /dev/null
+/*
+ * replicate.c - Replicate SiriDB.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/fifo.h>
+#include <siri/db/insert.h>
+#include <siri/db/replicate.h>
+#include <siri/db/series.h>
+#include <siri/err.h>
+#include <siri/net/protocol.h>
+#include <siri/siri.h>
+#include <stddef.h>
+
+#define REPLICATE_SLEEP 10 /* 10 milliseconds * active tasks */
+#define REPLICATE_TIMEOUT 300000 /* 5 minutes */
+
+static void REPLICATE_work(uv_timer_t * handle);
+static void REPLICATE_on_repl_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+static void REPLICATE_on_repl_finished_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+
+/*
+ * Return 0 is successful or -1 and a SIGNAL is raised if failed.
+ *
+ * replicate->status : REPLICATE_IDLE
+ */
+int siridb_replicate_init(siridb_t * siridb, siridb_initsync_t * initsync)
+{
+ assert (siri.loop != NULL);
+
+ siridb->replicate = malloc(sizeof(siridb_replicate_t));
+ if (siridb->replicate == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ siridb->replicate->initsync = initsync;
+
+ siridb->replicate->timer = malloc(sizeof(uv_timer_t));
+ if (siridb->replicate->timer == NULL)
+ {
+ ERR_ALLOC
+ siridb->replicate->status = REPLICATE_CLOSED;
+ siridb_replicate_free(&siridb->replicate);
+ return -1;
+ }
+ siridb->replicate->timer->data = siridb;
+
+ siridb->replicate->status = REPLICATE_IDLE;
+
+ uv_timer_init(siri.loop, siridb->replicate->timer);
+
+ return 0;
+}
+
+/*
+ * This will close the replicate task.
+ *
+ * Warning: make sure open promises are closed since the 'REPLICATE_work'
+ * depends on having siridb->replicate.
+ */
+void siridb_replicate_close(siridb_replicate_t * replicate)
+{
+ assert (replicate != NULL &&
+ replicate->timer != NULL &&
+ replicate->status != REPLICATE_CLOSED);
+ /* we can use uv_timer_stop() even if the timer is not scheduled */
+ uv_timer_stop(replicate->timer);
+ uv_close((uv_handle_t *) replicate->timer, (uv_close_cb) free);
+ replicate->status = REPLICATE_CLOSED;
+}
+
+/*
+ * This will destroy the replicate task.
+ * Make sure to call 'siridb_replicate_close' first.
+ *
+ * Warning: make sure open promises are closed since the 'REPLICATE_work'
+ * depends on having siridb->replicate.
+ */
+void siridb_replicate_free(siridb_replicate_t ** replicate)
+{
+ assert ((*replicate)->status == REPLICATE_CLOSED);
+ if ((*replicate)->initsync != NULL)
+ {
+ siridb_initsync_free(&(*replicate)->initsync);
+ }
+ free(*replicate);
+
+ *replicate = NULL;
+}
+
+/*
+ * Returns 0 if successful or anything else if not.
+ * (signal is set in case of an error)
+ */
+int siridb_replicate_pkg(siridb_t * siridb, sirinet_pkg_t * pkg)
+{
+ int rc = siridb_fifo_append(siridb->fifo, pkg);
+ if (!rc && siridb_replicate_is_idle(siridb->replicate))
+ {
+ siridb_replicate_start(siridb->replicate);
+ }
+ return rc;
+}
+
+/*
+ * Start replicate task. Only call this function when status is 'idle'.
+ * Idle status can be checked using 'siridb_replicate_is_idle(replicate)'
+ */
+void siridb_replicate_start(siridb_replicate_t * replicate)
+{
+ assert (siridb_replicate_is_idle(replicate));
+ replicate->status = REPLICATE_RUNNING;
+ if (replicate->initsync == NULL)
+ {
+ uv_timer_start(
+ replicate->timer,
+ REPLICATE_work,
+ REPLICATE_SLEEP,
+ 0);
+ }
+ else
+ {
+ siridb_initsync_run(replicate->timer);
+ }
+}
+
+/*
+ * Pause the replicate process. This can take some time and one should check
+ * for replicate->status to be REPLICATE_PAUSED. Do not stop the fifo buffer
+ * before the replication process is truly paused.
+ */
+void siridb_replicate_pause(siridb_replicate_t * replicate)
+{
+ assert (replicate->status != REPLICATE_CLOSED);
+ replicate->status = (replicate->status == REPLICATE_IDLE) ?
+ REPLICATE_PAUSED : REPLICATE_STOPPING;
+}
+
+/*
+ * Continue the replication process.
+ *
+ * (this will start the replicate task)
+ */
+void siridb_replicate_continue(siridb_replicate_t * replicate)
+{
+ /* make sure the fifo buffer is open */
+ assert (siridb_fifo_is_open(((siridb_t *) replicate->timer->data)->fifo));
+ assert (replicate->status != REPLICATE_CLOSED);
+
+ replicate->status = (replicate->status == REPLICATE_STOPPING) ?
+ REPLICATE_RUNNING : REPLICATE_IDLE;
+
+ if (replicate->initsync != NULL)
+ {
+ siridb_initsync_fopen(replicate->initsync, "r+");
+
+ if (replicate->initsync == NULL)
+ {
+ log_critical("Cannot open initial synchronization file: '%s'",
+ replicate->initsync->fn);
+ siridb_initsync_free(&replicate->initsync);
+ }
+ }
+
+ if (siridb_replicate_is_idle(replicate))
+ {
+ siridb_replicate_start(replicate);
+ }
+}
+
+
+
+/*
+ * This function can raise a SIGNAL.
+ */
+static void REPLICATE_work(uv_timer_t * handle)
+{
+ siridb_t * siridb = (siridb_t *) handle->data;
+ sirinet_pkg_t * pkg;
+
+ assert (siridb->fifo != NULL);
+ assert (siridb->replicate != NULL);
+ assert (siridb->replica != NULL);
+ assert (siridb->replicate->status != REPLICATE_IDLE);
+ assert (siridb->replicate->status != REPLICATE_PAUSED);
+ assert (siridb->replicate->status != REPLICATE_CLOSED);
+ assert (siridb->replicate->initsync == NULL);
+ assert (siridb_fifo_is_open(siridb->fifo));
+
+ if ( siridb->replicate->status == REPLICATE_RUNNING &&
+ siridb_fifo_has_data(siridb->fifo) &&
+ ( siridb_server_is_accessible(siridb->replica) ||
+ siridb_server_is_synchronizing(siridb->replica)) &&
+ (pkg = siridb_fifo_pop(siridb->fifo)) != NULL)
+ {
+ if (siridb_server_send_pkg(
+ siridb->replica,
+ pkg,
+ REPLICATE_TIMEOUT,
+ (sirinet_promise_cb) REPLICATE_on_repl_response,
+ siridb,
+ 0))
+ {
+ free(pkg);
+ }
+ }
+ else
+ {
+ if ( siridb_server_is_synchronizing(siridb->replica) &&
+ !siridb_fifo_has_data(siridb->fifo))
+ {
+ pkg = sirinet_pkg_new(0, 0, BPROTO_REPL_FINISHED, NULL);
+ if (pkg != NULL && siridb_server_send_pkg(
+ siridb->replica,
+ pkg,
+ 0,
+ (sirinet_promise_cb) REPLICATE_on_repl_finished_response,
+ NULL,
+ 0))
+ {
+ free(pkg);
+ }
+ }
+ siridb->replicate->status =
+ (siridb->replicate->status == REPLICATE_STOPPING) ?
+ REPLICATE_PAUSED : REPLICATE_IDLE;
+ }
+}
+
+/*
+ * Return a pkg without series which are scheduled for initial synchronization.
+ *
+ * In case of an error, NULL is returned and a SIGNAL is raised.
+ */
+sirinet_pkg_t * siridb_replicate_pkg_filter(
+ siridb_t * siridb,
+ unsigned char * data,
+ size_t len,
+ int flags)
+{
+ siridb_series_t * series;
+ qp_packer_t * netpacker = sirinet_packer_new(len + sizeof(sirinet_pkg_t));
+
+ if (netpacker == NULL)
+ {
+ return NULL; /*signal is raised */
+ }
+
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, data, len);
+
+ qp_obj_t qp_series_name;
+
+ qp_next(&unpacker, NULL); /* map */
+ qp_add_type(netpacker, QP_MAP_OPEN);
+
+ qp_next(&unpacker, &qp_series_name); /* first series or end */
+ while (qp_is_raw_term(&qp_series_name))
+ {
+ series = (siridb_series_t *) ct_get(
+ siridb->series,
+ (const char *) qp_series_name.via.raw);
+ if (series == NULL || (~series->flags & SIRIDB_SERIES_INIT_REPL))
+ {
+ /* raw is terminated so len is included a terminator char */
+ qp_add_raw(
+ netpacker,
+ qp_series_name.via.raw,
+ qp_series_name.len);
+ qp_packer_extend_fu(netpacker, &unpacker);
+ }
+ else
+ {
+ qp_skip_next(&unpacker);
+ }
+
+ qp_next(&unpacker, &qp_series_name);
+ }
+
+ sirinet_pkg_t * npkg = sirinet_packer2pkg(
+ netpacker,
+ 0,
+ (flags & INSERT_FLAG_TEST) ?
+ BPROTO_INSERT_TEST_SERVER : (flags & INSERT_FLAG_TESTED) ?
+ BPROTO_INSERT_TESTED_SERVER : BPROTO_INSERT_SERVER);
+
+ return npkg;
+}
+
+/*
+ * Call-back function: sirinet_promise_cb
+ */
+static void REPLICATE_on_repl_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ siridb_t * siridb = (siridb_t *) promise->data;
+
+ /* open promises must be closed before siridb->replicate is destroyed */
+ assert (siridb->replicate != NULL);
+ assert (siridb->fifo != NULL);
+
+ switch ((sirinet_promise_status_t) status)
+ {
+ case PROMISE_WRITE_ERROR:
+ /*
+ * Write to socket error, data is not send so we should not commit.
+ */
+ break;
+ case PROMISE_TIMEOUT_ERROR:
+ /*
+ * Timeout occurred, use commit error since the data can be
+ * processed by the replica, we're just not sure.
+ */
+ case PROMISE_CANCELLED_ERROR:
+ /*
+ * Promise is cancelled but most likely the data is successful
+ * processed. Use siridb_fifo_commit_err() since we're not sure.
+ */
+ case PROMISE_PKG_TYPE_ERROR:
+ /*
+ * Commit with error since this package has result in an unknown
+ * package type.
+ */
+ siridb_fifo_commit_err(siridb->fifo);
+ break;
+ case PROMISE_SUCCESS:
+ if (sirinet_protocol_is_error(pkg->tp))
+ {
+ log_error(
+ "Error occurred while processing data on the replica: "
+ "(response type: %u)", pkg->tp);
+ siridb_fifo_commit_err(siridb->fifo);
+ }
+ else
+ {
+ siridb_fifo_commit(siridb->fifo);
+ }
+ break;
+ }
+
+ if (siridb->replicate->status != REPLICATE_CLOSED)
+ {
+ uv_timer_start(
+ siridb->replicate->timer,
+ REPLICATE_work,
+ REPLICATE_SLEEP * siridb->tasks.active,
+ 0);
+ }
+ sirinet_promise_decref(promise);
+}
+
+static void REPLICATE_on_repl_finished_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ if (status)
+ {
+ /* we already have a log entry so this can be a debug log */
+ log_debug(
+ "Error while sending replication finished to '%s' (%s)",
+ promise->server->name,
+ sirinet_promise_strstatus(status));
+ }
+ else if (pkg->tp == BPROTO_ACK_REPL_FINISHED)
+ {
+ log_info("Replication finished ACK received from '%s'",
+ promise->server->name);
+ }
+ else
+ {
+ log_critical("Unexpected package type received from '%s' (type: %u)",
+ promise->server->name,
+ pkg->tp);
+ }
+
+ /* we must free the promise */
+ sirinet_promise_decref(promise);
+}
+
+
+
+
--- /dev/null
+/*
+ * series.c - SiriDB Time Series.
+ *
+ *
+ * Info siridb->series_mutex:
+ *
+ * Main thread:
+ * siridb->series_map : read (no lock) write (lock)
+ * series->idx : read (lock) write (lock)
+ *
+ * Other threads:
+ * siridb->series_map : read (lock) write (not allowed)
+ * series->idx : read (lock) write (lock)
+ *
+ * Note: One exception to 'not allowed' are the free functions
+ * since they only run when no other references to the object exist.
+ */
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <logger/logger.h>
+#include <siri/db/buffer.h>
+#include <siri/db/db.h>
+#include <siri/db/misc.h>
+#include <siri/db/series.h>
+#include <siri/db/shard.h>
+#include <siri/db/shards.h>
+#include <siri/err.h>
+#include <siri/siri.h>
+#include <string.h>
+#include <xpath/xpath.h>
+
+#define SIRIDB_SERIES_FN "series.dat"
+#define SIRIDB_CORRUPT_FN "corrupt_series.dat"
+#define SIRIDB_DROPPED_FN ".dropped"
+#define SIRIDB_MAX_SERIES_ID_FN ".max_series_id"
+#define SIRIDB_SERIES_SCHEMA 1
+#define DROPPED_DUMMY 1
+
+/*
+ * Used for storing double and integers as string. this is not very important
+ * if it will not store all characters generated so 64 is more than enough
+ */
+#define STR_TYPE_BUF_SZ 64
+static char str_type_buf[STR_TYPE_BUF_SZ];
+
+static int SERIES_save(siridb_t * siridb);
+static int SERIES_load(siridb_t * siridb, imap_t * dropped);
+static int SERIES_read_dropped(siridb_t * siridb, imap_t * dropped);
+static int SERIES_open_new_dropped_file(siridb_t * siridb);
+static int SERIES_open_dropped_file(siridb_t * siridb);
+static int SERIES_update_max_id(siridb_t * siridb);
+static void SERIES_update_start(siridb_series_t *__restrict series);
+static void SERIES_update_end(siridb_series_t *__restrict series);
+static void SERIES_update_overlap(siridb_series_t *__restrict series);
+static inline int SERIES_pack(siridb_series_t * series, qp_fpacker_t * fpacker);
+static void SERIES_idx_sort(
+ idx_t * idx,
+ uint_fast32_t start,
+ uint_fast32_t end);
+
+static siridb_series_t * SERIES_new(
+ siridb_t * siridb,
+ uint32_t id,
+ uint8_t tp,
+ uint16_t pool,
+ const char * name);
+
+const char series_type_map[3][8] = {
+ "integer",
+ "float",
+ "string"
+};
+
+const uint8_t SERIES_SFC =
+ SIRIDB_SHARD_HAS_NEW_VALUES | SIRIDB_SHARD_IS_LOADING;
+
+/*
+ * Call-back used to compare series properties.
+ *
+ * Returns 1 when series match or 0 if not.
+ */
+int siridb_series_cexpr_cb(siridb_series_t * series, cexpr_condition_t * cond)
+{
+ switch ((enum cleri_grammar_ids) cond->prop)
+ {
+ case CLERI_GID_K_LENGTH:
+ return cexpr_int_cmp(cond->operator, series->length, cond->int64);
+ case CLERI_GID_K_START:
+ return cexpr_int_cmp(cond->operator, series->start, cond->int64);
+ case CLERI_GID_K_END:
+ return cexpr_int_cmp(cond->operator, series->end, cond->int64);
+ case CLERI_GID_K_POOL:
+ return cexpr_int_cmp(cond->operator, series->pool, cond->int64);
+ case CLERI_GID_K_SHARD_DURATION:
+ return cexpr_int_cmp(
+ cond->operator,
+ (series->idx ? series->idx->shard->duration : 0),
+ cond->int64);
+ case CLERI_GID_K_TYPE:
+ return cexpr_int_cmp(cond->operator, series->tp, cond->int64);
+ case CLERI_GID_K_NAME:
+ return cexpr_str_cmp(cond->operator, series->name, cond->str);
+ default:
+ /* we must NEVER get here */
+ log_critical("Unexpected series property received: %d", cond->prop);
+ assert (0);
+ }
+ return -1;
+}
+
+void series_update_start_end(siridb_series_t * series)
+{
+ SERIES_update_start(series);
+ SERIES_update_end(series);
+}
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ *
+ * Warnings:
+ * - Do not call this function after a siri_err has occurred since this
+ * would risk a stack overflow in case this series has caused the siri_err
+ * and the buffer length is not reset.
+ *
+ * - This method will update the series->length but updating the time-stamps
+ * (series->start and series->end) should be done outside this function.
+ */
+int siridb_series_add_point(
+ siridb_t *__restrict siridb,
+ siridb_series_t *__restrict series,
+ uint64_t * ts,
+ qp_via_t * val)
+{
+ assert (!siri_err);
+ assert (series->buffer != NULL);
+ int rc = 0;
+
+ series->length++;
+
+ /* add point in memory
+ * (memory can hold 1 more point than we can hold on disk)
+ */
+ siridb_points_add_point(series->buffer, ts, val);
+
+ if (series->buffer->len == siridb->buffer->len)
+ {
+ if (siridb_shards_add_points(
+ siridb,
+ series,
+ series->buffer))
+ {
+ rc = -1; /* signal is raised */
+ }
+ else
+ {
+ series->buffer->len = 0;
+ if (siridb_buffer_write_empty(siridb->buffer, series))
+ {
+ ERR_FILE
+ rc = -1;
+ }
+ }
+ }
+ else
+ {
+ if (siridb_buffer_write_point(siridb->buffer, series, ts, val))
+ {
+ ERR_FILE
+ log_critical("Cannot write new point to buffer");
+ rc = -1;
+ }
+ }
+
+ return rc;
+}
+
+/*
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ *
+ * Warnings:
+ * - Do not call this function after a siri_err has occurred since this
+ * would risk a stack overflow in case this series has caused the siri_err
+ * and the buffer length is not reset.
+ *
+ * - This method will update the series->length but updating the time-stamps
+ * (series->start and series->end) should be done outside this function.
+ */
+int siridb_series_add_pcache(
+ siridb_t *__restrict siridb,
+ siridb_series_t *__restrict series,
+ siridb_pcache_t *__restrict pcache)
+{
+ if (pcache->len > siridb->buffer->len || series->buffer == NULL)
+ {
+ series->length += pcache->len;
+
+ return siridb_shards_add_points(
+ siridb,
+ series,
+ (siridb_points_t *) pcache);
+ }
+
+ if (pcache->len + series->buffer->len > siridb->buffer->len)
+ {
+ series->length += pcache->len;
+
+ siridb_points_t *__restrict points = series->buffer;
+ size_t i = points->len;
+ siridb_point_t *__restrict point;
+
+ while (i--)
+ {
+ point = points->data + i;
+ if (siridb_pcache_add_point(pcache, &point->ts, &point->val))
+ {
+ return -1; /* signal is raised */
+ }
+ }
+
+ if (siridb_shards_add_points(
+ siridb,
+ series,
+ (siridb_points_t *) pcache))
+ {
+ return -1; /* signal is raised */
+ }
+
+ series->buffer->len = 0;
+ if (siridb_buffer_write_empty(siridb->buffer, series))
+ {
+ ERR_FILE
+ return -1;
+ }
+ }
+ else
+ {
+ siridb_point_t *__restrict point;
+ size_t i;
+
+ for (i = 0; i < pcache->len; i++)
+ {
+ point = pcache->data + i;
+
+ if (siridb_series_add_point(
+ siridb,
+ series,
+ &point->ts,
+ &point->val))
+ {
+ return -1; /* signal is raised */
+ }
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ *
+ * This function adds the new series to siridb->series_map and siridb->series.
+ */
+siridb_series_t * siridb_series_new(
+ siridb_t * siridb,
+ const char * series_name,
+ uint8_t tp)
+{
+ siridb_series_t * series;
+
+ siridb->max_series_id++;
+ series = SERIES_new(
+ siridb,
+ siridb->max_series_id,
+ tp,
+ siridb->server->pool,
+ series_name);
+
+ if (series == NULL)
+ {
+ return NULL; /* signal is raised */
+ }
+ /* add series to the store */
+ if (qp_fadd_type(siridb->store, QP_ARRAY3) ||
+ qp_fadd_raw(
+ siridb->store,
+ (const unsigned char *) series_name,
+ series->name_len + 1) ||
+ qp_fadd_int64(siridb->store, (int64_t) series->id) ||
+ qp_fadd_int64(siridb->store, (int64_t) series->tp) ||
+ qp_flush(siridb->store))
+ {
+ ERR_FILE
+ log_critical("Cannot write series '%s' to store.", series_name);
+ siridb__series_free(series);
+ return NULL;
+ }
+
+ /* create a buffer for series (except string series) */
+ if (tp != TP_STRING && siridb_buffer_new_series(siridb->buffer, series))
+ {
+ /* signal is raised */
+ log_critical("Could not create buffer for series '%s'.",
+ series_name);
+ siridb__series_free(series);
+ return NULL;
+ }
+
+ if (imap_add(siridb->series_map, series->id, series))
+ {
+ log_critical("Error adding series '%s' to the internal imap.",
+ series_name);
+ siridb__series_free(series);
+ ERR_ALLOC
+ return NULL;
+ }
+
+ if (ct_add(siridb->series, series->name, series))
+ {
+ log_critical("Error adding series '%s' to the internal smap.",
+ series_name);
+ imap_pop(siridb->series_map, series->id);
+ siridb__series_free(series);
+ ERR_ALLOC
+ return NULL;
+ }
+
+ /* we can ignore the result code since this is not critical and logging
+ * is done by the function.
+ */
+ siridb_groups_add_series(siridb->groups, series);
+
+ return series;
+}
+
+/*
+ * NEVER call this function but call siridb_series_decref instead.
+ *
+ * Destroy series. (parsing NULL is not allowed)
+ */
+void siridb__series_free(siridb_series_t *__restrict series)
+{
+ siridb_shard_t * shard;
+ uint_fast32_t i;
+
+ /* mark shards with dropped series flag */
+ for (i = 0; i < series->idx_len; i++)
+ {
+ shard = series->idx[i].shard;
+ shard->flags |= SIRIDB_SHARD_HAS_DROPPED_SERIES;
+ siridb_shard_decref(shard);
+ }
+
+ if (series->buffer != NULL)
+ {
+ siridb_points_free(series->buffer);
+ if (series->flags & SIRIDB_SERIES_IS_DROPPED)
+ {
+ vec_append_safe(
+ &series->siridb->buffer->empty,
+ (void *) series->bf_offset);
+ }
+ }
+
+ free(series->idx);
+ free(series->name);
+ free(series);
+}
+
+/*
+ * Returns 0 if successful or -1 in case or an error.
+ * (a SIGNAL might be raised)
+ */
+int siridb_series_load(siridb_t * siridb)
+{
+ imap_t * dropped;
+
+ /* we must have a server because we need to know the pool id */
+ assert (siridb->server != NULL);
+ log_info("Loading series");
+
+ dropped = imap_new();
+
+ if (dropped == NULL)
+ {
+ return -1;
+ }
+
+ if (SERIES_read_dropped(siridb, dropped) ||
+ SERIES_load(siridb, dropped))
+ {
+ imap_free(dropped, NULL);
+ return -1;
+ }
+
+ imap_free(dropped, NULL);
+
+ if (SERIES_update_max_id(siridb) ||
+ SERIES_open_new_dropped_file(siridb) ||
+ siridb_series_open_store(siridb))
+ {
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * This function should only be called when new values are added.
+ * For example, during optimization we do not use this function for
+ * replacing indexes. This way we can set the HAS_NEW_VALUES correctly.
+ *
+ * Returns 0 if successful; -1 and a SIGNAL is raised in case an error occurred.
+ */
+int siridb_series_add_idx(
+ siridb_series_t *__restrict series,
+ siridb_shard_t *__restrict shard,
+ uint64_t start_ts,
+ uint64_t end_ts,
+ uint32_t pos,
+ uint16_t len,
+ uint16_t cinfo)
+{
+ idx_t * idx;
+ uint32_t i = series->idx_len;
+ series->idx_len++;
+
+ /* never zero */
+ idx = (idx_t *) realloc(
+ series->idx,
+ series->idx_len * sizeof(idx_t));
+ if (idx == NULL)
+ {
+ ERR_ALLOC
+ series->idx_len--;
+ return -1;
+ }
+ series->idx = idx;
+
+ for (; i && start_ts < series->idx[i - 1].start_ts; i--)
+ {
+ series->idx[i] = series->idx[i - 1];
+ }
+
+ idx = series->idx + i;
+
+ /* Do not set the new values check when shard is loading. */
+ if (!(shard->flags & SERIES_SFC))
+ {
+ shard->flags |= SIRIDB_SHARD_HAS_NEW_VALUES;
+ }
+
+ idx->start_ts = start_ts;
+ idx->end_ts = end_ts;
+ idx->len = len;
+ idx->shard = shard;
+ idx->pos = pos;
+ idx->cinfo = cinfo;
+
+ /* We do not have to save an overlap since it will be detected again when
+ * reading the shard at startup.
+ */
+ if ( (i > 0 &&
+ start_ts < series->idx[i - 1].end_ts) ||
+ (i + 1 < series->idx_len &&
+ end_ts > series->idx[i + 1].start_ts))
+ {
+ shard->flags |= SIRIDB_SHARD_HAS_OVERLAP;
+ series->flags |= SIRIDB_SERIES_HAS_OVERLAP;
+ }
+
+ siridb_shard_incref(shard);
+
+ return 0;
+}
+
+/*
+ * Remove series from the indexes and mark the series as 'dropped'.
+ *
+ * Do not forget to call 'siridb_series_drop_commit' to commit the drop.
+ */
+void siridb_series_drop_prepare(siridb_t * siridb, siridb_series_t * series)
+{
+ /* remove series from map */
+ imap_pop(siridb->series_map, series->id);
+
+ /* remove series from tree */
+ ct_pop(siridb->series, series->name);
+
+ series->flags |= SIRIDB_SERIES_IS_DROPPED;
+}
+
+/*
+ * Write dropped series to file and decrement series reference counter.
+ * Function 'siridb_series_drop_prepare' should be called before this function.
+ *
+ * In case we cannot write the series_id to the drop file we log critical and
+ * return -1 but no signal is raised.
+ *
+ * Warning: do not forget to call 'siridb_series_flush_dropped()'
+ *
+ * Return 0 if successful.
+ */
+int siridb_series_drop_commit(siridb_t * siridb, siridb_series_t * series)
+{
+ assert (series->flags & SIRIDB_SERIES_IS_DROPPED);
+
+ int rc = 0;
+
+ if (siridb->dropped_fp == NULL && SERIES_open_dropped_file(siridb))
+ {
+ rc = -1;
+ }
+ /* we are sure the pointer is at the end of the file */
+ else if (fwrite(&series->id, sizeof(uint32_t), 1, siridb->dropped_fp) != 1)
+ {
+ log_critical("Cannot write %d to dropped cache file.", series->id);
+ rc = -1;
+ };
+
+ /* decrement reference to series */
+ siridb_series_decref(series);
+
+ return rc;
+}
+
+/*
+ * In case we cannot write the series_id to the drop file we log critical but
+ * no other action is taken.
+ *
+ * Warning: do not forget to call 'siridb_series_flush_dropped()'
+ *
+ * Return 0 if successful or -1 in case the dropped series was not written
+ * to file. The series is still dropped from memory in the case.
+ *
+ * In case the series is flagged as 'dropped', 0 (successful) is returned.
+ */
+int siridb_series_drop(siridb_t * siridb, siridb_series_t * series)
+{
+ /*
+ * This function is used from build series maps in async functions.
+ * Therefore a series might already be dropped, for example by a second
+ * drop statement or the re-index task.
+ */
+ if (~series->flags & SIRIDB_SERIES_IS_DROPPED)
+ {
+ siridb_series_drop_prepare(siridb, series);
+ return siridb_series_drop_commit(siridb, series);
+ }
+ return 0;
+}
+
+/*
+ * Return 0 if successful or EOF if not.
+ *
+ * Logging is done but no error is raised in case of failure.
+ *
+ * This function flushes the dropped file and sets the dropped series flag
+ * for groups update thread.
+ */
+int siridb_series_flush_dropped(siridb_t * siridb)
+{
+ int rc = 0;
+
+ if (siridb->dropped_fp == NULL && SERIES_open_dropped_file(siridb))
+ {
+ rc = -1;
+ }
+ else if (fflush(siridb->dropped_fp))
+ {
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_DROPPED_FN)
+ log_critical("Could not flush dropped file: '%s'", fn);
+ rc = -1;
+ }
+
+ siridb->groups->flags |= GROUPS_FLAG_DROPPED_SERIES;
+ siridb->tags->flags |= TAGS_FLAG_DROPPED_SERIES;
+
+ return rc;
+}
+
+/*
+ * Re-allocations in this function can fail but are not critical.
+ *
+ * Note that 'series' can be destroyed when series->length has reached zero.
+ */
+void siridb_series_remove_shard(
+ siridb_t *__restrict siridb,
+ siridb_series_t *__restrict series,
+ siridb_shard_t *__restrict shard)
+{
+ idx_t *__restrict idx;
+ uint_fast32_t i, offset;
+ uint64_t start = shard->id - series->mask;
+ uint64_t end = start + shard->duration;
+
+ i = offset = 0;
+
+ for ( idx = series->idx;
+ i < series->idx_len;
+ i++, idx++)
+ {
+ if (idx->shard == shard)
+ {
+ siridb_shard_decref(shard);
+ offset++;
+ series->length -= idx->len;
+ }
+ else if (offset)
+ {
+ series->idx[i - offset] = series->idx[i];
+ }
+ }
+
+ if (offset)
+ {
+ if (!series->length)
+ {
+ series->idx_len = 0;
+
+ if (siridb_series_drop(siridb, series))
+ {
+ siridb_series_flush_dropped(siridb);
+ }
+ }
+ else
+ {
+ series->idx_len -= offset;
+ idx = (idx_t *) realloc(
+ series->idx,
+ series->idx_len * sizeof(idx_t));
+ if (idx == NULL && series->idx_len)
+ {
+ log_error("Re-allocation failed while removing series from "
+ "shard index");
+ }
+ else
+ {
+ series->idx = idx;
+ }
+ if (series->start >= start && series->start < end)
+ {
+ SERIES_update_start(series);
+ }
+ if (series->end < end && series->end > start)
+ {
+ SERIES_update_end(series);
+ }
+ }
+ }
+}
+
+/*
+ * Update series properties.
+ *
+ * (series might be destroyed if series->length is zero)
+ */
+void siridb_series_update_props(siridb_t * siridb, siridb_series_t * series)
+{
+ if (series->tp != TP_STRING && series->buffer == NULL)
+ {
+ log_error(
+ "Drop '%s' (%" PRIu32 ") since no buffer is found for this series",
+ series->name,
+ series->id);
+ siridb_series_drop(siridb, series);
+ }
+ else
+ {
+ SERIES_update_start(series);
+ SERIES_update_end(series);
+
+ if (!series->length)
+ {
+ log_warning(
+ "Drop '%s' (%" PRIu32 ") since no data is found for this series",
+ series->name,
+ series->id);
+ siridb_series_drop(siridb, series);
+ }
+ }
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+siridb_points_t * siridb_series_get_points(
+ siridb_series_t *__restrict series,
+ uint64_t *__restrict start_ts,
+ uint64_t *__restrict end_ts)
+{
+ idx_t *__restrict idx;
+ siridb_points_t *__restrict points;
+ siridb_point_t *__restrict point;
+ size_t len, size;
+ uint32_t i;
+ uint32_t indexes[series->idx_len];
+ len = i = size = 0;
+
+ for ( idx = series->idx;
+ i < series->idx_len;
+ i++, idx++)
+ {
+ if ( (start_ts == NULL || idx->end_ts >= *start_ts) &&
+ (end_ts == NULL || idx->start_ts < *end_ts))
+ {
+ size += idx->len;
+ indexes[len] = i;
+ len++;
+ }
+ }
+
+ size += (series->buffer == NULL) ? 0 : series->buffer->len;
+ points = siridb_points_new(size, series->tp);
+
+ if (points == NULL)
+ {
+ ERR_ALLOC /* TODO: maybe remove ERR_ALLOC */
+ return NULL;
+ }
+
+ for (i = 0; i < len; i++)
+ {
+ idx = series->idx + indexes[i];
+ siridb_shard_get_points_callback(idx->shard->flags, series)(
+ points,
+ idx,
+ start_ts,
+ end_ts,
+ series->flags & SIRIDB_SERIES_HAS_OVERLAP);
+ /* errors can be ignored here */
+ }
+
+ if (series->buffer != NULL)
+ {
+ /* create pointer to buffer and get current length */
+ point = series->buffer->data;
+ len = series->buffer->len;
+
+ /* crop start buffer if needed */
+ if (start_ts != NULL)
+ {
+ for (; len && point->ts < *start_ts; point++, len--);
+ }
+
+ /* crop end buffer if needed */
+ if (end_ts != NULL && len)
+ {
+ siridb_point_t *__restrict p;
+
+ for ( p = point + len - 1;
+ len && p->ts >= *end_ts;
+ p--, len--);
+ }
+
+ /* add buffer points */
+ for (; len; point++, len--)
+ {
+ siridb_points_add_point(points, &point->ts, &point->val);
+ }
+ }
+
+ if (points->len < size && siridb_points_resize(points, points->len))
+ {
+ log_error("Re-allocation points has failed");
+ }
+
+ return points;
+}
+
+/*
+ * Can be used instead of the macro function when need as callback function.
+ */
+void siridb__series_decref(siridb_series_t * series)
+{
+ siridb_series_decref(series);
+}
+
+siridb_points_t * siridb_series_get_first(
+ siridb_series_t * series, int * required_shard)
+{
+ siridb_points_t * buf = series->buffer;
+ siridb_points_t * points;
+ uint64_t start;
+
+ if (buf != NULL &&
+ buf->len &&
+ buf->data->ts == series->start)
+ {
+ points = siridb_points_new(1, series->tp);
+ if (points == NULL)
+ {
+ return NULL;
+ }
+
+ /* string type does not have a buffer so we don't have to worry */
+ points->data->ts = buf->data->ts;
+ points->data->val = buf->data->val;
+ points->len = 1;
+ return points;
+ }
+
+ (*required_shard)++;
+
+ /* if not in the buffer, then if must be in a shard */
+ assert (series->idx_len);
+
+ idx_t * first = series->idx;
+
+ points = siridb_points_new(first->len, series->tp);
+ if (points == NULL)
+ {
+ return NULL;
+ }
+
+ start = series->start + 1;
+
+ siridb_shard_get_points_callback(first->shard->flags, series)(
+ points,
+ first,
+ NULL,
+ &start,
+ series->flags & SIRIDB_SERIES_HAS_OVERLAP);
+
+ /* we must have at least one point, more points are possible when
+ * having multiple points at the same time-stamp. */
+ assert (points->len);
+
+ while (points->len > 1)
+ {
+ --points->len;
+ if (points->tp == TP_STRING)
+ {
+ free((points->data + points->len)->val.str);
+ }
+ }
+
+ return points;
+}
+
+siridb_points_t * siridb_series_get_last(
+ siridb_series_t * series, int * required_shard)
+{
+ siridb_points_t * buf = series->buffer;
+ siridb_points_t * points;
+ siridb_point_t * point;
+
+ if (buf != NULL &&
+ buf->len &&
+ (point = buf->data + (buf->len - 1))->ts == series->end)
+ {
+ points = siridb_points_new(1, series->tp);
+ if (points == NULL)
+ {
+ return NULL;
+ }
+
+ /* string type does not have a buffer so we don't have to worry */
+ points->data->ts = point->ts;
+ points->data->val = point->val;
+ points->len = 1;
+ return points;
+ }
+
+ (*required_shard)++;
+
+ /* if not in the buffer, then if must be in a shard */
+ assert (series->idx_len);
+
+ /* if not in the buffer, then if must be in a shard */
+
+ size_t i = series->idx_len - 1;
+ idx_t * idx = series->idx + i;
+ idx_t * last = idx;
+
+ for (; i && last->shard == (--idx)->shard; --i)
+ {
+ if (idx->end_ts > last->end_ts)
+ {
+ last = idx;
+ }
+ }
+
+ points = siridb_points_new(last->len, series->tp);
+ if (points == NULL)
+ {
+ return NULL;
+ }
+
+ siridb_shard_get_points_callback(last->shard->flags, series)(
+ points,
+ last,
+ &last->end_ts,
+ NULL,
+ series->flags & SIRIDB_SERIES_HAS_OVERLAP);
+
+ /* we must have at least one point, more points are possible when
+ * having multiple points at the same time-stamp. */
+ assert (points->len);
+
+ while (points->len > 1)
+ {
+ --points->len;
+ if (points->tp == TP_STRING)
+ {
+ free((points->data + points->len)->val.str);
+ }
+ }
+
+ return points;
+}
+
+siridb_points_t * siridb_series_get_count(siridb_series_t * series)
+{
+ siridb_points_t * points = siridb_points_new(1, TP_INT);
+ if (points != NULL)
+ {
+ points->data->ts = series->end;
+ points->data->val.int64 = series->length;
+ points->len = 1;
+ }
+ return points;
+}
+
+void siridb_series_ensure_type(siridb_series_t * series, qp_obj_t * qp_obj)
+{
+ switch(series->tp)
+ {
+ case TP_INT:
+ if (qp_obj->tp != QP_INT64)
+ {
+ if (qp_obj->tp == QP_DOUBLE)
+ {
+ double d = qp_obj->via.real;
+ qp_obj->via.int64 = (int64_t) d;
+ }
+ else if (qp_obj->tp == QP_RAW)
+ {
+ char * s = strndup(qp_obj->via.str, qp_obj->len);
+ qp_obj->via.int64 = \
+ (s == NULL) ? 0 : (int64_t) strtoll(s, NULL, 10);
+ free(s);
+ }
+ else
+ {
+ assert(0);
+ }
+ qp_obj->tp = QP_INT64;
+ }
+ return;
+ case TP_DOUBLE:
+ if (qp_obj->tp != QP_DOUBLE)
+ {
+ if (qp_obj->tp == QP_INT64)
+ {
+ int64_t i = qp_obj->via.int64;
+ qp_obj->via.real = (double) i;
+ }
+ else if (qp_obj->tp == QP_RAW)
+ {
+ char * s = strndup(qp_obj->via.str, qp_obj->len);
+ qp_obj->via.real = \
+ (s == NULL) ? 0.0 : strtod(s, NULL);
+ free(s);
+ }
+ else
+ {
+ assert(0);
+ }
+ qp_obj->tp = QP_DOUBLE;
+ }
+ return;
+ case TP_STRING:
+ if (qp_obj->tp != QP_RAW)
+ {
+ if (qp_obj->tp == QP_INT64)
+ {
+ qp_obj->len = snprintf(
+ str_type_buf,
+ STR_TYPE_BUF_SZ,
+ "%" PRId64,
+ qp_obj->via.int64);
+ qp_obj->via.str = str_type_buf;
+ }
+ else if (qp_obj->tp == QP_DOUBLE)
+ {
+ qp_obj->len = snprintf(
+ str_type_buf,
+ STR_TYPE_BUF_SZ,
+ "%f",
+ qp_obj->via.real);
+ qp_obj->via.str = str_type_buf;
+ }
+ else
+ {
+ assert(0);
+ }
+ qp_obj->tp = QP_RAW;
+ }
+ return;
+ }
+ assert (0);
+}
+
+/*
+ * Calculate the server id.
+ * Returns 0 or 1, representing a server in a pool)
+ */
+uint8_t siridb_series_server_id_by_name(const char * name)
+{
+ uint32_t n;
+
+ /* get sum series name to calculate series mask (for sharding) */
+ for (n = 0; *name; name++)
+ {
+ n += *name;
+ }
+
+ return (uint8_t) ((n / 11) % 2);
+}
+
+/*
+ * Returns 0 if successful or -1 and a SIGNAL is raised in case of a critical
+ * error.
+ * Note that we also return 0 if we had to recover a shard. In this case you
+ * can find the errors we had to recover in the log file. (log level should
+ * be at least 'ERROR' for all error logs)
+ */
+int siridb_series_optimize_shard(
+ siridb_t *__restrict siridb,
+ siridb_series_t *__restrict series,
+ siridb_shard_t *__restrict shard)
+{
+ idx_t *__restrict idx;
+
+ uint_fast32_t i, start, end, new_idx;
+ uint64_t max_ts;
+ size_t size;
+ siridb_points_t *__restrict points;
+ int rc;
+ uint16_t cinfo = 0;
+ max_ts = (shard->id + shard->duration) - series->mask;
+
+ rc = new_idx = end = i = size = start = 0;
+
+ for ( idx = series->idx;
+ i < series->idx_len && idx->start_ts < max_ts;
+ i++, idx++)
+ {
+ if (idx->shard == shard->replacing)
+ {
+ if (!end)
+ {
+ end = start = i;
+ }
+ size += idx->len;
+ end++;
+
+ /*
+ * we have at least 2 references to the shard so we never
+ * reach 0 here. (this ref + optimize ref)
+ */
+ siridb_shard_decref(shard->replacing);
+ }
+ else if (idx->shard == shard && end)
+ {
+ new_idx++;
+ }
+ }
+
+ if (!end)
+ {
+ /* no data for this series is found in the shard */
+ return rc;
+ }
+
+ end += new_idx;
+
+ size_t pos;
+ uint16_t chunk_sz;
+ uint_fast32_t num_chunks, pstart, pend, diff;
+ siridb_shard_get_points_cb get_points_cb = \
+ siridb_shard_get_points_callback(shard->replacing->flags, series);
+
+ points = siridb_points_new(size, series->tp);
+ if (points == NULL)
+ {
+ /* TODO: check if we can remove this ERR_ALLOC */
+ ERR_ALLOC
+ return -1;
+ }
+
+ for (i = start; i < end; i++)
+ {
+ idx = series->idx + i;
+ /* we can have indexes for this 'new' shard which we should skip */
+ if (idx->shard == shard->replacing && get_points_cb(
+ points,
+ idx,
+ NULL,
+ NULL,
+ series->flags & SIRIDB_SERIES_HAS_OVERLAP))
+ {
+ /* an error occurred while reading points, logging is done */
+ size -= idx->len;
+ }
+ }
+
+ num_chunks = (size - 1) / shard->max_chunk_sz + 1;
+ chunk_sz = size / num_chunks + (size % num_chunks != 0);
+ i = start;
+
+ for (pstart = 0; pstart < size; pstart += chunk_sz)
+ {
+ pend = pstart + chunk_sz;
+ if (pend > size)
+ {
+ pend = size;
+ }
+
+ if ((pos = siridb_shard_write_points(
+ siridb,
+ series,
+ shard,
+ points,
+ pstart,
+ pend,
+ siri.optimize->idx_fp,
+ &cinfo)) == 0)
+ {
+ log_critical(
+ "Cannot write points to shard id '%" PRIu64 "'",
+ shard->id);
+ rc = -1; /* signal is raised */
+ num_chunks--;
+ }
+ else
+ {
+ /*
+ * We should always find a spot for this index since the number
+ * of chunks cannot grow.
+ */
+ do
+ {
+ idx = series->idx + i;
+ i++;
+ }
+ while (idx->shard == shard);
+
+ assert (idx->shard == shard->replacing);
+
+ idx->shard = shard;
+ idx->start_ts = points->data[pstart].ts;
+ idx->end_ts = points->data[pend - 1].ts;
+ idx->len = pend - pstart;
+ idx->pos = pos;
+ idx->cinfo = cinfo;
+ siridb_shard_incref(shard);
+ }
+ }
+
+ siridb_points_free(points);
+
+ if (new_idx)
+ {
+ /*
+ * We might have skipped new_indexes while writing new blocks and
+ * possible some new_indexes exist at the wrong place in the index.
+ *
+ * Therefore we must sort the series index part containing data
+ * for this shard.
+ */
+ SERIES_idx_sort(series->idx, start, end - 1);
+
+ /*
+ * We need to set 'i' to the correct value since 'i' has possible
+ * not walked over all 'new indexes'.
+ *
+ * (in case new_idx is 0, i is already equal to the value set below)
+ */
+ i = start + new_idx + num_chunks;
+ }
+
+ if (i < end)
+ {
+ /* get the difference */
+ diff = end - i;
+
+ /* new length is current length minus difference */
+ series->idx_len -= diff;
+
+ for (; i < series->idx_len; i++)
+ {
+ series->idx[i] = series->idx[i + diff];
+ }
+
+ /* shrink memory to the new size */
+ idx = (idx_t *) realloc(
+ series->idx,
+ series->idx_len * sizeof(idx_t));
+ if (idx == NULL && series->idx_len)
+ {
+ /* this is not critical since the original allocated block still
+ * works.
+ */
+ log_error("Shrinking memory for one series has failed!");
+ }
+ else
+ {
+ series->idx = idx;
+ }
+ }
+ else
+ {
+ /* start must be equal to end if not smaller */
+ assert (i == end);
+ }
+
+ if (series->flags & SIRIDB_SERIES_HAS_OVERLAP)
+ {
+ SERIES_update_overlap(series);
+ }
+
+ return rc;
+}
+
+/*
+ * Open SiriDB series store file.
+ *
+ * Returns 0 if successful or -1 in case of an error.
+ */
+int siridb_series_open_store(siridb_t * siridb)
+{
+ /* macro get series file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_SERIES_FN)
+
+ if ((siridb->store = qp_open(fn, "a")) == NULL)
+ {
+ log_critical("Cannot open file '%s' for appending", fn);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * Will sort an index to its correct order. The start of idx should be correct
+ * with a valid shard. All replaced shard indexes are sorted towards the end.
+ */
+static void SERIES_idx_sort(
+ idx_t * idx,
+ uint_fast32_t start,
+ uint_fast32_t end)
+{
+ siridb_shard_t * shard = idx[start].shard;
+ idx_t * a, * b;
+ uint_fast32_t i = start;
+ uint_fast32_t n, m;
+ idx_t tmp;
+
+ /*
+ * Since the first position is always correct we can leave this one alone.
+ */
+ while (++i < end)
+ {
+ a = idx + i;
+ b = idx + i + 1;
+ /*
+ * We only need to swap if at least the last position is of the correct
+ * shard and if the time-stamps should be swapped.
+ */
+ if ( b->shard == shard &&
+ (a->shard != shard || a->start_ts > b->start_ts))
+ {
+ /*
+ * Swap at least a and b but also check if we can swap a - 1 with b
+ */
+ n = i - start;
+ m = i + 1;
+ tmp = idx[m];
+ do
+ {
+ idx[m] = idx[m - 1];
+ m--; /* we must decrement here */
+ }
+ while (--n && (
+ idx[m - 1].shard != tmp.shard ||
+ idx[m - 1].start_ts > tmp.start_ts));
+ idx[m] = tmp;
+ }
+ }
+}
+
+/*
+ * Updates series->flags and remove SIRIDB_SERIES_HAS_OVERLAP if possible.
+ * This function never sets an overlap and therefore should not be called
+ * as long as the overlap flag is not set.
+ */
+static void SERIES_update_overlap(siridb_series_t *__restrict series)
+{
+ uint_fast32_t i;
+
+ assert (series->flags & SIRIDB_SERIES_HAS_OVERLAP);
+
+ for (i = 1; i < series->idx_len; i++)
+ {
+ if (series->idx[i - 1].end_ts > series->idx[i].start_ts)
+ {
+ return;
+ }
+ }
+ series->flags &= ~SIRIDB_SERIES_HAS_OVERLAP;
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+static siridb_series_t * SERIES_new(
+ siridb_t * siridb,
+ uint32_t id,
+ uint8_t tp,
+ uint16_t pool,
+ const char * name)
+{
+ uint32_t n;
+ siridb_series_t * series = malloc(sizeof(siridb_series_t));
+ if (series == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ series->name = strdup(name);
+ if (series->name == NULL)
+ {
+ ERR_ALLOC
+ free(series);
+ series = NULL;
+ }
+ else
+ {
+ /* we use the length a lot and we have room so store this info */
+ series->name_len = strlen(series->name);
+ series->id = id;
+ series->tp = tp;
+ series->ref = 1;
+ series->length = 0;
+ series->start = -1;
+ series->end = 0;
+ series->buffer = NULL;
+ series->pool = pool;
+ series->flags = 0;
+ series->idx_len = 0;
+ series->idx = NULL;
+ series->siridb = siridb;
+
+ /* get sum series name to calculate series mask (for sharding) */
+ for (n = 0; *name; name++)
+ {
+ n += *name;
+ }
+
+ series->mask = (tp == TP_STRING) ?
+ (uint16_t) ((n / 11) % siridb->shard_mask_log) + 600 :
+ (uint16_t) ((n / 11) % siridb->shard_mask_num);
+
+ if ((_Bool) ((n / 11) % 2))
+ {
+ series->flags |= SIRIDB_SERIES_IS_SERVER_ONE;
+ }
+
+ /* make sure these two are exactly the same */
+ assert (siridb_series_server_id(series) ==
+ siridb_series_server_id_by_name(series->name));
+
+ if (siridb->time->precision == SIRIDB_TIME_SECONDS)
+ {
+ series->flags |= SIRIDB_SERIES_IS_32BIT_TS;
+ }
+ }
+ }
+ return series;
+}
+
+/*
+ * Raises a SIGNAL in case or an error.
+ *
+ * Returns always 0 but the result will be ignored since this function is used
+ * in ct_walk().
+ */
+static inline int SERIES_pack(siridb_series_t * series, qp_fpacker_t * fpacker)
+{
+ return (qp_fadd_type(fpacker, QP_ARRAY3) ||
+ qp_fadd_raw(
+ fpacker,
+ (unsigned char *) series->name,
+ series->name_len + 1) ||
+ qp_fadd_int64(fpacker, (int64_t) series->id) ||
+ qp_fadd_int64(fpacker, (int64_t) series->tp));
+}
+
+/*
+ * Returns 0 if successful or a negative integer in case of an error.
+ * (SIGNAL is raised in case of an error)
+ */
+static int SERIES_save(siridb_t * siridb)
+{
+ qp_fpacker_t * fpacker;
+
+ log_debug("Cleanup series file");
+
+ /* macro get series file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_SERIES_FN)
+
+ if ((fpacker = qp_open(fn, "w")) == NULL)
+ {
+ ERR_FILE
+ log_critical("Cannot open file '%s' for writing", fn);
+ return EOF;
+ }
+
+
+ if (/* open a new array */
+ qp_fadd_type(fpacker, QP_ARRAY_OPEN) ||
+
+ /* write the current schema */
+ qp_fadd_int64(fpacker, SIRIDB_SERIES_SCHEMA))
+ {
+ ERR_FILE
+ }
+ else
+ {
+ if (imap_walk(siridb->series_map, (imap_cb) &SERIES_pack, fpacker))
+ {
+ ERR_FILE
+ }
+ }
+ /* close file pointer */
+ if (qp_close(fpacker))
+ {
+ ERR_FILE
+ }
+
+ return siri_err;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * (a SIGNAL might be raised but -1 should be considered critical in any case)
+ */
+static int SERIES_read_dropped(siridb_t * siridb, imap_t * dropped)
+{
+ char * buffer;
+ char * pt;
+ long int size;
+ int rc = 0;
+ FILE * fp;
+
+ log_debug("Loading dropped series");
+
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_DROPPED_FN)
+
+ if ((fp = fopen(fn, "r")) == NULL)
+ {
+ /* no drop file, we have nothing to do */
+ return 0;
+ }
+
+ /* get file size */
+ if (fseeko(fp, 0, SEEK_END) ||
+ (size = ftello(fp)) < 0 ||
+ fseeko(fp, 0, SEEK_SET))
+ {
+ fclose(fp);
+ log_critical("Cannot read size of file '%s'", fn);
+ rc = -1;
+ }
+ else if (size)
+ {
+
+ buffer = malloc(size);
+ if (buffer == NULL)
+ {
+ log_critical("Cannot allocate buffer for reading dropped series");
+ rc = -1;
+ }
+ else if (fread(buffer, size, 1, fp) == 1)
+ {
+ char * end = buffer + size;
+ for ( pt = buffer;
+ pt < end;
+ pt += sizeof(uint32_t))
+ {
+ if (imap_set(
+ dropped,
+ (uint32_t) *((uint32_t *) pt),
+ (int *) DROPPED_DUMMY) == -1)
+ {
+ log_critical("Cannot add id to dropped map");
+ rc = -1;
+ }
+ }
+ }
+ else
+ {
+ log_critical("Cannot read %ld bytes from file '%s'", size, fn);
+ rc = -1;
+ }
+ free(buffer);
+ }
+
+ fclose(fp);
+
+ return rc;
+}
+
+static int SERIES_keep_corrupt_series(siridb_t * siridb)
+{
+ int rc;
+ siridb_misc_get_fn(series_fn, siridb->dbpath, SIRIDB_SERIES_FN)
+ siridb_misc_get_fn(corrupt_fn, siridb->dbpath, SIRIDB_CORRUPT_FN)
+
+ (void) unlink(corrupt_fn);
+ rc = rename(series_fn, corrupt_fn);
+ if (rc == 0)
+ {
+ log_warning("Keep previous '%s' as '%s'", series_fn, corrupt_fn);
+ }
+ else
+ {
+ log_error("Cannot rename '%s' to '%s'", series_fn, corrupt_fn);
+ }
+ return rc;
+}
+
+static int SERIES_load(siridb_t * siridb, imap_t * dropped)
+{
+ qp_unpacker_t * unpacker;
+ qp_obj_t qp_series_name;
+ qp_obj_t qp_series_id;
+ qp_obj_t qp_series_tp;
+ siridb_series_t * series;
+ qp_types_t tp;
+ uint32_t series_id;
+ uint8_t series_tp;
+
+ /* we should not have any series at this moment */
+ assert(siridb->max_series_id == 0);
+
+ /* get series file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_SERIES_FN)
+
+ if (!xpath_file_exist(fn))
+ {
+ /* missing series file, create an empty file and return */
+ return SERIES_save(siridb);
+ }
+
+ if ((unpacker = qp_unpacker_ff(fn)) == NULL)
+ {
+ return -1;
+ }
+
+ /* unpacker will be freed in case schema check fails */
+ siridb_misc_schema_check(SIRIDB_SERIES_SCHEMA)
+
+ while (qp_next(unpacker, NULL) == QP_ARRAY3 &&
+ qp_next(unpacker, &qp_series_name) == QP_RAW &&
+ qp_next(unpacker, &qp_series_id) == QP_INT64 &&
+ qp_next(unpacker, &qp_series_tp) == QP_INT64)
+ {
+ series_id = (uint32_t) qp_series_id.via.int64;
+
+ /* update max_series_id */
+ if (series_id > siridb->max_series_id)
+ {
+ siridb->max_series_id = series_id;
+ }
+
+ if (imap_get(dropped, series_id) == NULL)
+ {
+ series_tp = (uint8_t) qp_series_tp.via.int64;
+ series = SERIES_new(
+ siridb,
+ series_id,
+ series_tp,
+ siridb->server->pool,
+ (const char *) qp_series_name.via.raw);
+
+ if (series != NULL)
+ {
+ /* add series to c-tree */
+ int rc = ct_add(siridb->series, series->name, series);
+
+ if (rc == CT_EXISTS)
+ {
+ /* Duplicate series found */
+ siridb_series_t * other = ct_get(
+ siridb->series,
+ series->name);
+
+ log_error(
+ "Series '%s' with ID %"PRIu32" has a duplicate "
+ "ID %"PRIu32", "
+ "(SiriDB will keep the highest ID)",
+ series->name,
+ series->id,
+ other->id);
+
+ if (other->id >= series->id)
+ {
+ siridb__series_free(series);
+ continue;
+ }
+
+ (void) ct_pop(siridb->series, series->name);
+ (void) imap_pop(siridb->series_map, other->id);
+
+ siridb__series_free(other);
+
+ rc = ct_add(siridb->series, series->name, series);
+ }
+
+ if(rc || imap_add(siridb->series_map, series->id, series))
+ {
+ log_critical("series cannot be added");
+ return -1;
+ }
+ }
+ }
+ }
+
+ /* save last object, should be QP_END */
+ tp = qp_next(unpacker, NULL);
+
+ if (tp != QP_END)
+ {
+ double start = (double) (unpacker->end - unpacker->source);
+ double pos = (double) (unpacker->pt - unpacker->source);
+
+ if (pos / start < 0.8)
+ {
+ log_critical(
+ "Cannot read at least 80 percent of '%s', "
+ "do not continue as this leads to data loss",
+ fn);
+ /* free unpacker */
+ qp_unpacker_ff_free(unpacker);
+ return -1;
+ }
+
+ log_error(
+ "Expected end of file '%s'; "
+ "Create a backup and continue", fn);
+
+ (void) SERIES_keep_corrupt_series(siridb);
+ }
+
+ /* free unpacker */
+ qp_unpacker_ff_free(unpacker);
+
+ /*
+ * In case of a siri_err we should not return -1;
+ * overwrite series because the
+ * file then might be incomplete.
+ */
+ if (siri_err || SERIES_save(siridb))
+ {
+ log_critical("Cannot write series index to disk");
+ return -1; /* signal is raised */
+ }
+
+ return siri_err;
+}
+
+/*
+ * Open a new SiriDB drop series file.
+ *
+ * Returns 0 if successful or -1 in case of an error.
+ */
+static int SERIES_open_new_dropped_file(siridb_t * siridb)
+{
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_DROPPED_FN)
+
+ if ((siridb->dropped_fp = fopen(fn, "w")) == NULL)
+ {
+ log_critical("Cannot open '%s' for writing", fn);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * Open SiriDB drop series file.
+ *
+ * Returns 0 if successful or -1 in case of an error.
+ */
+static int SERIES_open_dropped_file(siridb_t * siridb)
+{
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_DROPPED_FN)
+
+ if ((siridb->dropped_fp = fopen(fn, "a")) == NULL)
+ {
+ log_critical("Cannot open '%s' for appending", fn);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * When series are dropped, the store still has this series so when
+ * SiriDB starts the next time we will include this dropped series by
+ * counting the max_series_id. A second restart could be a problem if
+ * not all shards are optimized because now the store does not have the
+ * last removed series and therefore the max_series_id could be set to
+ * a value for which shards still have data. Creating a new series and
+ * another SiriDB restart before the optimize has finished could lead
+ * to problems.
+ *
+ * Saving max_series_id at startup solves this issue because it will
+ * include the dropped series.
+ *
+ * Returns 0 if successful or -1 in case of an error.
+ */
+static int SERIES_update_max_id(siridb_t * siridb)
+{
+ int rc = 0;
+ FILE * fp;
+ uint32_t max_series_id = 0;
+
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_MAX_SERIES_ID_FN)
+
+ if ((fp = fopen(fn, "r")) != NULL)
+ {
+ if (fread(&max_series_id, sizeof(uint32_t), 1, fp) != 1)
+ {
+ log_critical("Cannot read max_series_id from '%s'", fn);
+ fclose(fp);
+ return -1;
+ }
+
+ if (fclose(fp))
+ {
+ log_critical("Cannot close max_series_id file: '%s'", fn);
+ return -1;
+ }
+
+ if (max_series_id > siridb->max_series_id)
+ {
+ siridb->max_series_id = max_series_id;
+ }
+ }
+
+ /* we only need to write max_series_id in case the one in the file is
+ * smaller or does not exist and max_series_id is larger than zero.
+ */
+ if (max_series_id < siridb->max_series_id)
+ {
+ if ((fp = fopen(fn, "w")) == NULL)
+ {
+ log_critical("Cannot open file '%s' for writing", fn);
+ return -1;
+ }
+
+ log_debug("Write max series id (%" PRIu32 ")", siridb->max_series_id);
+
+ if (fwrite(&siridb->max_series_id, sizeof(uint32_t), 1, fp) != 1)
+ {
+ log_critical("Cannot write max_series_id to file '%s'", fn);
+ rc = -1;
+ }
+
+ if (fclose(fp))
+ {
+ log_critical("Cannot save max_series_id to file '%s'", fn);
+ rc = -1;
+ }
+ }
+ return rc;
+}
+
+/*
+ * Update series 'start' property.
+ */
+static void SERIES_update_start(siridb_series_t *__restrict series)
+{
+ series->start = series->idx_len ? series->idx->start_ts : UINT64_MAX;
+
+ if (series->buffer && series->buffer->len)
+ {
+ siridb_point_t * point = series->buffer->data;
+ if (point->ts < series->start)
+ {
+ series->start = point->ts;
+ }
+ }
+}
+
+/*
+ * Update series 'end' property.
+ */
+static void SERIES_update_end(siridb_series_t *__restrict series)
+{
+ if (series->idx_len)
+ {
+ uint64_t start = 0;
+ idx_t * idx;
+ uint_fast32_t i;
+
+ for (i = series->idx_len; i--;)
+ {
+ idx = series->idx + i;
+
+ if (idx->end_ts < start)
+ {
+ break;
+ }
+
+ start = idx->start_ts;
+ if (idx->end_ts > series->end)
+ {
+ series->end = idx->end_ts;
+ }
+ }
+ }
+ else
+ {
+ series->end = 0;
+ }
+
+ if (series->buffer && series->buffer->len)
+ {
+ siridb_point_t * point = series->buffer->data +
+ series->buffer->len - 1;
+ if (point->ts > series->end)
+ {
+ series->end = point->ts;
+ }
+ }
+}
+
+
--- /dev/null
+/*
+ * server.c - Each SiriDB database has at least one server.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <procinfo/procinfo.h>
+#include <siri/db/query.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/db/fifo.h>
+#include <siri/db/tee.h>
+#include <siri/err.h>
+#include <siri/net/promise.h>
+#include <siri/net/stream.h>
+#include <siri/net/tcp.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <xstr/xstr.h>
+#include <string.h>
+
+#define SIRIDB_SERVERS_FN "servers.dat"
+#define SIRIDB_SERVERS_SCHEMA 1
+#define SIRIDB_SERVER_FLAGS_TIMEOUT 5000 /* 5 seconds */
+#define SIRIDB_SERVER_PROMISES_QUEUE_SIZE 250 /* max concurrent promises */
+#define FMT_AS_IPV6(addr) (strchr(addr, ':') != NULL)
+
+static int SERVER_update_name(siridb_server_t * server);
+static void SERVER_timeout_pkg(uv_timer_t * handle);
+static void SERVER_write_cb(uv_write_t * req, int status);
+static void SERVER_on_auth_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+static void SERVER_on_flags_update_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status);
+static void SERVER_on_connect(uv_connect_t * req, int status);
+static void SERVER_on_resolved(
+ uv_getaddrinfo_t * resolver,
+ int status,
+ struct addrinfo * res);
+static int SERVER_resolve_dns(
+ siridb_server_t * server,
+ int ai_family,
+ uv_getaddrinfo_cb getaddrinfo_cb);
+static void SERVER_on_data(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void SERVER_cancel_promise(sirinet_promise_t * promise);
+static void SERVER_upd_flag_queue_full(siridb_server_t * server);
+
+/*
+ * In case of an error the return value is NULL and a SIGNAL is raised.
+ */
+siridb_server_t * siridb_server_new(
+ const char * uuid,
+ const char * address,
+ size_t address_len,
+ uint16_t port,
+ uint16_t pool)
+{
+ siridb_server_t * server = malloc(sizeof(siridb_server_t));
+ if (server == NULL)
+ {
+ ERR_ALLOC
+ return NULL;
+ }
+ /* copy uuid */
+ memcpy(server->uuid, uuid, 16);
+
+ /* initialize with NULL, SERVER_update_name() sets the correct name */
+ server->name = NULL;
+
+ /* copy address */
+ server->address = strndup(address, address_len);
+ if (server->address == NULL)
+ {
+ ERR_ALLOC
+ free(server);
+ return NULL;
+ }
+
+ server->port = port;
+ server->pool = pool;
+ server->flags = 0;
+ server->id = 255;
+ server->ref = 0;
+ server->pid = 0;
+ server->version = NULL;
+ server->ip_support = 255; /* unknown */
+ server->retry_attempts = 0;
+ server->libuv = NULL;
+ server->dbpath = NULL;
+ server->buffer_path = NULL;
+ server->buffer_size = 0;
+ server->startup_time = 0;
+
+ /* we set the promises later because we don't need one for self */
+ server->promises = NULL;
+ server->client = NULL;
+
+ /* sets address:port to name property */
+ if (SERVER_update_name(server))
+ {
+ ERR_ALLOC
+ siridb__server_free(server);
+ server = NULL;
+ }
+
+ return server;
+}
+
+/*
+ * This function can return -1 and raise a SIGNAL which means the 'cb'
+ * function will NOT be called. Usually this function should return 0 which
+ * means that we try to send the package and the 'cb' function can be checked
+ * for the result.
+ *
+ * Note that 'pkg->pid' will be overwritten with a new package id.
+ *
+ * (default timeout PROMISE_DEFAULT_TIMEOUT is used when timeout 0 is set)
+ *
+ */
+int siridb_server_send_pkg(
+ siridb_server_t * server,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promise_cb cb,
+ void * data,
+ int flags)
+{
+ assert (server->client != NULL);
+ assert (server->promises != NULL);
+ assert (cb != NULL);
+ int rc;
+ uint8_t n = 0;
+ sirinet_promise_t * promise = malloc(sizeof(sirinet_promise_t));
+ if (promise == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ promise->timer = malloc(sizeof(uv_timer_t));
+ if (promise->timer == NULL)
+ {
+ ERR_ALLOC
+ free(promise);
+ return -1;
+ }
+ promise->timer->data = promise;
+ promise->cb = cb;
+ promise->pkg = (flags & FLAG_KEEP_PKG) ? NULL : pkg;
+ promise->ref = 2;
+ /*
+ * we do not need to increment the server reference counter since promises
+ * will be destroyed before the server is destroyed.
+ */
+ promise->server = server;
+ promise->data = data;
+
+ uv_write_t * req = malloc(sizeof(uv_write_t));
+ if (req == NULL)
+ {
+ ERR_ALLOC
+ free(promise->timer);
+ free(promise);
+ return -1;
+ }
+
+ while (++n)
+ {
+ /*
+ * Usually the first attempt is fine but in some rare case the pid
+ * might still be in use and we should try another pid.
+ */
+ promise->pid = server->pid++;
+ rc = imap_add(server->promises, promise->pid, promise);
+
+ if (rc == 0)
+ {
+ SERVER_upd_flag_queue_full(server);
+ break;
+ }
+
+ if (rc == -1)
+ {
+ /* memory allocation error */
+ free(promise->timer);
+ free(promise);
+ free(req);
+ ERR_ALLOC
+ return -1;
+ }
+
+ /* rc == -2, pid in use, try next pid */
+ }
+
+ if (!n)
+ {
+ /*
+ * The queue cannot contain more than SIRIDB_SERVER_PROMISES_QUEUE_SIZE
+ * promises so we should always find a free pid and this code should
+ * never be reached.
+ */
+ log_critical("Cannot add promise to queue for '%s'", server->name);
+ ERR_C
+ free(promise->timer);
+ free(promise);
+ free(req);
+ return -1;
+ }
+
+ pkg->pid = promise->pid;
+
+ uv_timer_init(siri.loop, promise->timer);
+ uv_timer_start(
+ promise->timer,
+ SERVER_timeout_pkg,
+ (timeout) ? timeout : PROMISE_DEFAULT_TIMEOUT,
+ 0);
+
+ log_debug("Sending (pid: %" PRIu16 ", len: %" PRIu32 ", tp: %s) to '%s'",
+ pkg->pid,
+ pkg->len,
+ sirinet_bproto_client_str(pkg->tp),
+ server->name);
+
+ req->data = promise;
+
+ /* set the correct check bit */
+ pkg->checkbit = pkg->tp ^ 255;
+
+ uv_buf_t wrbuf = uv_buf_init(
+ (char *) pkg,
+ sizeof(sirinet_pkg_t) + pkg->len);
+
+ uv_write(
+ req,
+ server->client->stream,
+ &wrbuf,
+ 1,
+ SERVER_write_cb);
+
+ return 0;
+}
+
+/*
+ * Register and return a new server from qpack data.
+ * The qpack data should contain: [uuid, address, port, pool]
+ *
+ * In case of an error NULL is returned.
+ * (a SIGNAL might be raised in case of allocation errors)
+ */
+siridb_server_t * siridb_server_register(
+ siridb_t * siridb,
+ unsigned char * data,
+ size_t len)
+{
+ siridb_server_t * server = NULL;
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, data, len);
+
+ qp_obj_t qp_uuid;
+ qp_obj_t qp_address;
+ qp_obj_t qp_port;
+ qp_obj_t qp_pool;
+
+ if (qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_uuid) == QP_RAW &&
+ qp_next(&unpacker, &qp_address) == QP_RAW &&
+ qp_next(&unpacker, &qp_port) == QP_INT64 &&
+ qp_next(&unpacker, &qp_pool) == QP_INT64)
+ {
+ server = siridb_server_new(
+ (const char *) qp_uuid.via.raw,
+ (const char *) qp_address.via.raw,
+ qp_address.len,
+ qp_port.via.int64,
+ qp_pool.via.int64);
+
+
+ if (server != NULL)
+ {
+ if ( (server->promises = imap_new()) == NULL ||
+ siridb_servers_register(siridb, server))
+ {
+ siridb__server_free(server);
+ server = NULL;
+ }
+ }
+ }
+
+ return server;
+}
+
+/*
+ * This function can raise a SIGNAL.
+ */
+void siridb_server_send_flags(siridb_server_t * server)
+{
+
+ assert (server->client != NULL);
+ assert (siridb_server_is_online(server));
+
+ sirinet_stream_t * client = server->client;
+
+ assert (client->siridb != NULL);
+
+ int16_t n = client->siridb->server->flags;
+ QP_PACK_INT16(buffer, n)
+
+ sirinet_pkg_t * pkg = sirinet_pkg_new(0, 3, BPROTO_FLAGS_UPDATE, buffer);
+ if (pkg != NULL && siridb_server_send_pkg(
+ server,
+ pkg,
+ SIRIDB_SERVER_FLAGS_TIMEOUT,
+ (sirinet_promise_cb) SERVER_on_flags_update_response,
+ NULL,
+ 0))
+ {
+ free(pkg);
+ }
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * (a SIGNAL might be raises)
+ *
+ * This function only updates the address/port/name in case different from the
+ * current values.
+ */
+int siridb_server_update_address(
+ siridb_t * siridb,
+ siridb_server_t * server,
+ const char * address,
+ uint16_t port)
+{
+ if (strcmp(server->address, address) || server->port != port)
+ {
+ char * tmp;
+ tmp = strdup(address);
+ if (tmp == NULL)
+ {
+ log_critical(
+ "Cannot set server address (memory allocation error)");
+ return -1;
+ }
+
+ free(server->address);
+
+ server->address = tmp;
+ server->port = port;
+
+ if FMT_AS_IPV6(server->address)
+ {
+ log_warning("Update server '%s' to '[%s]:%u'",
+ server->name,
+ server->address,
+ server->port);
+ }
+ else
+ {
+ log_warning("Update server '%s' to '%s:%u'",
+ server->name,
+ server->address,
+ server->port);
+ }
+ if (SERVER_update_name(server))
+ {
+ log_critical(
+ "Cannot rename server (memory allocation error)");
+ return -1;
+ }
+
+ if (siridb_servers_save(siridb))
+ {
+ return -1;
+ }
+
+ return 1;
+ }
+
+ return 0;
+}
+
+/*
+ * Connect to a SiriDB Server.
+ */
+void siridb_server_connect(siridb_t * siridb, siridb_server_t * server)
+{
+ /* server->socket must be NULL at this point */
+ assert (server->client == NULL);
+
+ ++server->retry_attempts;
+ server->client = sirinet_stream_new(STREAM_TCP_SERVER, &SERVER_on_data);
+
+ if (server->client != NULL)
+ {
+ struct in_addr sa;
+ struct in6_addr sa6;
+ server->client->origin = server;
+ server->client->siridb = siridb;
+ siridb_incref(siridb);
+ siridb_server_incref(server);
+ uv_tcp_init(siri.loop, (uv_tcp_t *) server->client->stream);
+
+ if (inet_pton(AF_INET, server->address, &sa))
+ {
+ /* IPv4 */
+ struct sockaddr_in dest;
+
+ uv_connect_t * req = malloc(sizeof(uv_connect_t));
+ if (req == NULL)
+ {
+ ERR_ALLOC
+ sirinet_stream_decref(server->client);
+ }
+ else
+ {
+ log_debug("Trying to connect to '%s'...", server->name);
+ uv_ip4_addr(server->address, server->port, &dest);
+ uv_tcp_connect(
+ req,
+ (uv_tcp_t *) server->client->stream,
+ (const struct sockaddr *) &dest,
+ SERVER_on_connect);
+ }
+ }
+ else if (inet_pton(AF_INET6, server->address, &sa6))
+ {
+ /* IPv6 */
+ struct sockaddr_in6 dest6;
+
+ uv_connect_t * req = malloc(sizeof(uv_connect_t));
+ if (req == NULL)
+ {
+ ERR_ALLOC
+ sirinet_stream_decref(server->client);
+ }
+ else
+ {
+ log_debug("Trying to connect to '%s'...", server->name);
+ uv_ip6_addr(server->address, server->port, &dest6);
+ uv_tcp_connect(
+ req,
+ (uv_tcp_t *) server->client->stream,
+ (const struct sockaddr *) &dest6,
+ SERVER_on_connect);
+ }
+ }
+ else
+ {
+ /* Try DNS */
+ if (SERVER_resolve_dns(
+ server,
+ dns_req_family_map(siri.cfg->ip_support),
+ SERVER_on_resolved))
+ {
+ sirinet_stream_decref(server->client);
+ }
+ }
+ }
+}
+
+/*
+ * Try to get an ip address from dns.
+ *
+ * The callback should be checked if resolving succeeded. This function should
+ * return 0 when we can start an attempt. When the result is not zero, the
+ * callback will not be called.
+ */
+static int SERVER_resolve_dns(
+ siridb_server_t * server,
+ int ai_family,
+ uv_getaddrinfo_cb getaddrinfo_cb)
+{
+
+ struct addrinfo hints;
+ hints.ai_family = ai_family;
+ hints.ai_socktype = SOCK_STREAM;
+ hints.ai_protocol = IPPROTO_TCP;
+ hints.ai_flags = AI_NUMERICSERV;
+
+ uv_getaddrinfo_t * resolver = malloc(sizeof(uv_getaddrinfo_t));
+
+ if (resolver == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ int result;
+ resolver->data = server;
+
+ char port[6]= {'\0'};
+ sprintf(port, "%u", server->port);
+
+ result = uv_getaddrinfo(
+ siri.loop,
+ resolver,
+ getaddrinfo_cb,
+ server->address,
+ port,
+ &hints);
+
+ if (result)
+ {
+ log_error("getaddrinfo call error %s", uv_err_name(result));
+ free(resolver);
+ }
+
+ return result;
+}
+
+/*
+ * Callback used to check if resolving an ip address was successful.
+ */
+static void SERVER_on_resolved(
+ uv_getaddrinfo_t * resolver,
+ int status,
+ struct addrinfo * res)
+{
+ siridb_server_t * server = resolver->data;
+
+ if (status < 0)
+ {
+ log_error("Cannot resolve ip address for server '%s' (error: %s)",
+ server->name,
+ uv_err_name(status));
+
+ sirinet_stream_decref(server->client);
+ }
+ else
+ {
+ if (Logger.level == LOGGER_DEBUG)
+ {
+ char addr[47] = {'\0'}; /* enough for both ipv4 and ipv6 */
+
+ switch (res->ai_family)
+ {
+ case AF_INET:
+ uv_ip4_name((struct sockaddr_in *) res->ai_addr, addr, 16);
+ break;
+
+ case AF_INET6:
+ uv_ip6_name((struct sockaddr_in6 *) res->ai_addr, addr, 46);
+ break;
+
+ default:
+ sprintf(addr, "unsupported family");
+ }
+
+ log_debug(
+ "Resolved ip address '%s' for server '%s', "
+ "trying to connect...",
+ addr, server->name);
+
+ }
+ uv_connect_t * req = malloc(sizeof(uv_connect_t));
+ if (req == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ uv_tcp_connect(
+ req,
+ (uv_tcp_t *) server->client->stream,
+ (const struct sockaddr *) res->ai_addr,
+ SERVER_on_connect);
+ }
+ }
+
+ uv_freeaddrinfo(res);
+ free(resolver);
+}
+
+/*
+ * Update SERVER_FLAG_QUEUE_FULL based on number of promises.
+ */
+static void SERVER_upd_flag_queue_full(siridb_server_t * server)
+{
+ if (server->promises->len >= SIRIDB_SERVER_PROMISES_QUEUE_SIZE)
+ {
+ server->flags |= SERVER_FLAG_QUEUE_FULL;
+ }
+ else
+ {
+ server->flags &= ~SERVER_FLAG_QUEUE_FULL;
+ }
+}
+
+/*
+ * Write call-back.
+ */
+static void SERVER_write_cb(uv_write_t * req, int status)
+{
+ sirinet_promise_t * promise = (sirinet_promise_t *) req->data;
+
+ if (status)
+ {
+ log_error(
+ "Socket write error to server '%s' (pid: %" PRIu16 ", error: %s)",
+ promise->server->name,
+ promise->pid,
+ uv_strerror(status));
+
+ if (imap_pop(promise->server->promises, promise->pid) == NULL)
+ {
+ log_critical(
+ "Got a socket error but the promise is not found. "
+ "(PID: %" PRIu16 ")",
+ promise->pid);
+ return;
+ }
+ SERVER_upd_flag_queue_full(promise->server);
+
+ uv_timer_stop(promise->timer);
+ uv_close((uv_handle_t *) promise->timer, (uv_close_cb) free);
+
+ promise->cb(promise, NULL, PROMISE_WRITE_ERROR);
+ }
+
+ free(promise->pkg); /* NULL when FLAG_KEEP_PKG is set */
+ sirinet_promise_decref(promise);
+
+ free(req);
+}
+
+/*
+ * Timeout received.
+ */
+static void SERVER_timeout_pkg(uv_timer_t * handle)
+{
+ sirinet_promise_t * promise = handle->data;
+
+ if (imap_pop(promise->server->promises, promise->pid) == NULL)
+ {
+ log_critical(
+ "Timeout task is called on package (PID %" PRIu16
+ ") for server '%s' "
+ "but we cannot find this promise!!",
+ promise->pid,
+ promise->server->name);
+ return;
+ }
+ else
+ {
+ SERVER_upd_flag_queue_full(promise->server);
+ log_warning("Timeout on package (PID %" PRIu16 ") for server '%s'",
+ promise->pid,
+ promise->server->name);
+ }
+ /* the timer is stopped but we still need to close the timer */
+ uv_close((uv_handle_t *) promise->timer, (uv_close_cb) free);
+
+ promise->cb(promise, NULL, PROMISE_TIMEOUT_ERROR);
+}
+
+/*
+ * This function can raise a SIGNAL.
+ */
+static void SERVER_on_connect(uv_connect_t * req, int status)
+{
+ sirinet_stream_t * client = req->handle->data;
+ siridb_t * siridb = client->siridb;
+ siridb_server_t * server = client->origin;
+
+ if (status == 0)
+ {
+ log_debug(
+ "Connection created to back-end server: '%s', "
+ "sending authentication request", server->name);
+
+ server->retry_attempts = 0; /* reset connection attempts */
+
+ uv_read_start(
+ req->handle,
+ sirinet_stream_alloc_buffer,
+ sirinet_stream_on_data);
+
+ sirinet_pkg_t * pkg;
+ qp_packer_t * packer = sirinet_packer_new(512);
+
+ if (packer == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ if (qp_add_type(packer, QP_ARRAY_OPEN) ||
+ qp_add_raw(packer, (const unsigned char *)
+ client->siridb->server->uuid, 16) ||
+ qp_add_string_term(packer, siridb->dbname) ||
+ qp_add_int64(packer, siridb->server->flags) ||
+ qp_add_string_term(packer, SIRIDB_VERSION) ||
+ qp_add_string_term(packer, SIRIDB_MINIMAL_VERSION) ||
+ qp_add_int64(packer, (int64_t) siri.cfg->ip_support) ||
+ qp_add_string_term(packer, uv_version_string()) ||
+ qp_add_string_term(packer, siridb->dbpath) ||
+ qp_add_string_term(packer, siridb->buffer->path) ||
+ qp_add_int64(packer, (int64_t) siridb->buffer->size) ||
+ qp_add_int64(packer, (int64_t) siri.startup_time) ||
+ qp_add_string_term(packer, siridb->server->address) ||
+ qp_add_int64(packer, (int64_t) siridb->server->port))
+ {
+ qp_packer_free(packer);
+ }
+ else
+ {
+ pkg = sirinet_packer2pkg(packer, 0, BPROTO_AUTH_REQUEST);
+
+ if (siridb_server_send_pkg(
+ server,
+ pkg,
+ 0,
+ (sirinet_promise_cb) SERVER_on_auth_response,
+ NULL,
+ 0))
+ {
+ free(pkg);
+ }
+ }
+ }
+ }
+ else
+ {
+ log_error("Connecting to back-end server '%s' failed (error: %s)",
+ server->name,
+ uv_strerror(status));
+
+ sirinet_stream_decref(client);
+ }
+ free(req);
+}
+
+/*
+ * on-data call-back function.
+ *In case the promise is found, promise->cb() will be called.
+ */
+static void SERVER_on_data(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ siridb_server_t * server = client->origin;
+ sirinet_promise_t * promise = imap_pop(server->promises, pkg->pid);
+
+ log_debug(
+ "Response received (pid: %" PRIu16
+ ", len: %" PRIu32 ", tp: %s) from '%s'",
+ pkg->pid,
+ pkg->len,
+ sirinet_bproto_server_str(pkg->tp),
+ server->name);
+
+ if (promise == NULL)
+ {
+ log_warning(
+ "Received a package (PID %" PRIu16
+ ") from server '%s' which has probably timed-out earlier.",
+ pkg->pid,
+ server->name);
+ }
+ else
+ {
+ SERVER_upd_flag_queue_full(promise->server);
+ uv_timer_stop(promise->timer);
+ uv_close((uv_handle_t *) promise->timer, (uv_close_cb) free);
+ promise->cb(promise, pkg, PROMISE_SUCCESS);
+ }
+}
+
+/*
+ * Returns the current server status (flags) as string. the returned value
+ * is created with malloc() so do not forget to free the result.
+ *
+ * NULL is returned in case malloc has failed.
+ */
+char * siridb_server_str_status(siridb_server_t * server)
+{
+ /* we must initialize the buffer according to the longest possible value */
+ char buffer[128] = {};
+ int n = 0;
+ int i;
+
+ for (i = 1; i < SERVER_FLAG_AUTHENTICATED; i *= 2)
+ {
+ if (server->flags & i)
+ {
+ switch (i)
+ {
+ case SERVER_FLAG_RUNNING:
+ strcat(buffer, (!n) ? "running" : " | " "running");
+ break;
+ case SERVER_FLAG_SYNCHRONIZING:
+ strcat(buffer, (!n) ? "synchronizing" : " | " "synchronizing");
+ break;
+ case SERVER_FLAG_REINDEXING:
+ strcat(buffer, (!n) ? "re-indexing" : " | " "re-indexing");
+ break;
+ case SERVER_FLAG_BACKUP_MODE:
+ strcat(buffer, (!n) ? "backup-mode" : " | " "backup-mode");
+ break;
+ case SERVER_FLAG_QUEUE_FULL:
+ strcat(buffer, (!n) ? "queue-full" : " | " "queue-full");
+ break;
+ case SERVER_FLAG_UNAVAILABLE:
+ strcat(buffer, (!n) ? "unavailable" : " | " "unavailable");
+ break;
+ }
+ n = 1;
+ }
+ }
+ return strdup((n) ? buffer : "offline");
+}
+
+/*
+ * Returns server object by node or NULL if the server is not found.
+ */
+siridb_server_t * siridb_server_from_node(
+ siridb_t * siridb,
+ cleri_node_t * server_node,
+ char * err_msg)
+{
+ siridb_server_t * server = NULL;
+
+ switch (server_node->cl_obj->tp)
+ {
+ case CLERI_TP_CHOICE: /* server name */
+ {
+ char name[server_node->len - 1];
+ xstr_extract_string(name, server_node->str, server_node->len);
+ server = siridb_servers_by_name(siridb->servers, name);
+ }
+ break;
+
+ case CLERI_TP_REGEX: /* uuid */
+ {
+ uuid_t uuid;
+ char * str_uuid = strndup(server_node->str, server_node->len);
+ server = (uuid_parse(str_uuid, uuid) == 0) ?
+ siridb_servers_by_uuid(siridb->servers, uuid) : NULL;
+ free(str_uuid);
+ }
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ if (server == NULL && err_msg != NULL)
+ {
+ snprintf(
+ err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot find server: %.*s",
+ (int) server_node->len,
+ server_node->str);
+ }
+
+ return server;
+}
+
+/*
+ * Returns 0 if successful or -1 and a SIGNAL is raised in case of an error.
+ * (an error can only happen when we are not able to save the new servers file)
+ */
+int siridb_server_drop(siridb_t * siridb, siridb_server_t * server)
+{
+ int rc = 0;
+ assert (siridb->server != server);
+ siridb_pool_t * pool = siridb->pools->pool + server->pool;
+
+ switch (server->id)
+ {
+ case 0:
+ assert (pool->len == 2);
+ pool->server[0] = pool->server[1];
+ pool->server[0]->id = 0;
+ /* FALLTHRU */
+ /* fall through */
+ case 1:
+ pool->server[1] = NULL;
+ pool->len = 1;
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ if (server == siridb->replica)
+ {
+ if (siridb->replicate != NULL)
+ {
+ siridb_replicate_close(siridb->replicate);
+ siridb_replicate_free(&siridb->replicate);
+ }
+
+ if (siridb->fifo != NULL)
+ {
+ siridb_fifo_free(siridb->fifo);
+ siridb->fifo = NULL;
+ }
+
+ siridb->replica = NULL;
+
+ if (siridb->server->flags & SERVER_FLAG_SYNCHRONIZING)
+ {
+ siridb->server->flags &= ~SERVER_FLAG_SYNCHRONIZING;
+ siridb_servers_send_flags(siridb->servers);
+ }
+ }
+
+ if ((siridb_server_t *) llist_remove(
+ siridb->servers, NULL, server) == server)
+ {
+ siridb_server_decref(server);
+ rc = siridb_servers_save(siridb);
+ }
+ else
+ {
+ /* can never be reached */
+ assert (0);
+ }
+ return rc;
+}
+
+/*
+ * Do not call this function but use siridb_server_decref.
+ *
+ * Free server. If the server has promises, each promise will be cancelled and
+ * so each promise->cb() will be called.
+ */
+void siridb__server_free(siridb_server_t * server)
+{
+ /* we MUST first free the promises because each promise has a reference to
+ * this server and the promise callback might depend on this.
+ */
+ if (server->promises != NULL)
+ {
+ imap_free(server->promises, (imap_free_cb) SERVER_cancel_promise);
+ }
+ free(server->name);
+ free(server->address);
+ free(server->version);
+ free(server->libuv);
+ free(server->dbpath);
+ free(server->buffer_path);
+ free(server);
+}
+
+/*
+ * Returns true when the given property (CLERI keyword) needs a remote query
+ */
+int siridb_server_is_remote_prop(uint32_t prop)
+{
+ switch (prop)
+ {
+ /* these are the local props */
+ case CLERI_GID_K_ADDRESS:
+ case CLERI_GID_K_BUFFER_PATH:
+ case CLERI_GID_K_BUFFER_SIZE:
+ case CLERI_GID_K_DBPATH:
+ case CLERI_GID_K_IP_SUPPORT:
+ case CLERI_GID_K_LIBUV:
+ case CLERI_GID_K_NAME:
+ case CLERI_GID_K_ONLINE:
+ case CLERI_GID_K_POOL:
+ case CLERI_GID_K_PORT:
+ case CLERI_GID_K_STARTUP_TIME:
+ case CLERI_GID_K_STATUS:
+ case CLERI_GID_K_UUID:
+ case CLERI_GID_K_VERSION:
+ return 0;
+ }
+ return 1;
+}
+
+int siridb_server_cexpr_cb(
+ siridb_server_walker_t * wserver,
+ cexpr_condition_t * cond)
+{
+ switch (cond->prop)
+ {
+ case CLERI_GID_K_ADDRESS:
+ return cexpr_str_cmp(
+ cond->operator,
+ wserver->server->address,
+ cond->str);
+
+ case CLERI_GID_K_BUFFER_PATH:
+ return cexpr_str_cmp(
+ cond->operator,
+ (wserver->siridb->server == wserver->server) ?
+ wserver->siridb->buffer->path :
+ (wserver->server->buffer_path != NULL) ?
+ wserver->server->buffer_path : "",
+ cond->str);
+
+ case CLERI_GID_K_BUFFER_SIZE:
+ return cexpr_int_cmp(
+ cond->operator,
+ (wserver->siridb->server == wserver->server) ?
+ wserver->siridb->buffer->size :
+ wserver->server->buffer_size,
+ cond->int64);
+
+ case CLERI_GID_K_DBPATH:
+ return cexpr_str_cmp(
+ cond->operator,
+ (wserver->siridb->server == wserver->server) ?
+ wserver->siridb->dbpath :
+ (wserver->server->dbpath != NULL) ?
+ wserver->server->dbpath : "",
+ cond->str);
+
+ case CLERI_GID_K_IP_SUPPORT:
+ return cexpr_str_cmp(
+ cond->operator,
+ sirinet_tcp_ip_support_str(
+ (wserver->siridb->server == wserver->server) ?
+ siri.cfg->ip_support :
+ wserver->server->ip_support),
+ cond->str);
+
+ case CLERI_GID_K_LIBUV:
+ return cexpr_str_cmp(
+ cond->operator,
+ (wserver->siridb->server == wserver->server) ?
+ uv_version_string() :
+ (wserver->server->libuv != NULL) ?
+ wserver->server->libuv : "",
+ cond->str);
+
+ case CLERI_GID_K_NAME:
+ return cexpr_str_cmp(
+ cond->operator,
+ wserver->server->name,
+ cond->str);
+
+ case CLERI_GID_K_ONLINE:
+ return cexpr_bool_cmp(
+ cond->operator,
+ ( wserver->siridb->server == wserver->server ||
+ wserver->server->client != NULL),
+ cond->int64);
+
+ case CLERI_GID_K_POOL:
+ return cexpr_int_cmp(
+ cond->operator,
+ wserver->server->pool,
+ cond->int64);
+
+ case CLERI_GID_K_PORT:
+ return cexpr_int_cmp(
+ cond->operator,
+ wserver->server->port,
+ cond->int64);
+
+ case CLERI_GID_K_STARTUP_TIME:
+ return cexpr_int_cmp(
+ cond->operator,
+ (wserver->siridb->server == wserver->server) ?
+ siri.startup_time :
+ wserver->server->startup_time,
+ cond->int64);
+
+ case CLERI_GID_K_STATUS:
+ {
+ char * status = siridb_server_str_status(wserver->server);
+ int rc = cexpr_str_cmp(cond->operator, status, cond->str);
+ free(status);
+ return rc;
+ }
+
+ case CLERI_GID_K_UUID:
+ {
+ char uuid[37];
+ uuid_unparse_lower(wserver->server->uuid, uuid);
+ return cexpr_str_cmp(cond->operator, uuid, cond->str);
+ }
+
+ case CLERI_GID_K_VERSION:
+ return cexpr_str_cmp(
+ cond->operator,
+ (wserver->siridb->server == wserver->server) ?
+ SIRIDB_VERSION :
+ (wserver->server->version != NULL) ?
+ wserver->server->version : "",
+ cond->str);
+
+ /* all properties below are 'remote properties'. if a remote property
+ * is detected we should perform the query on each server and only for
+ * that specific server.
+ */
+ case CLERI_GID_K_LOG_LEVEL:
+ return cexpr_int_cmp(
+ cond->operator,
+ Logger.level,
+ cond->int64);
+
+ case CLERI_GID_K_MAX_OPEN_FILES:
+ return cexpr_int_cmp(
+ cond->operator,
+ siri.cfg->max_open_files,
+ cond->int64);
+
+ case CLERI_GID_K_MEM_USAGE:
+ return cexpr_int_cmp(
+ cond->operator,
+ (int64_t) (procinfo_total_physical_memory() / 1024),
+ cond->int64);
+
+ case CLERI_GID_K_FIFO_FILES:
+ return cexpr_int_cmp(
+ cond->operator,
+ (int64_t) siridb_fifo_size(wserver->siridb->fifo),
+ cond->int64);
+
+ case CLERI_GID_K_OPEN_FILES:
+ return cexpr_int_cmp(
+ cond->operator,
+ siridb_open_files(wserver->siridb),
+ cond->int64);
+
+ case CLERI_GID_K_RECEIVED_POINTS:
+ return cexpr_int_cmp(
+ cond->operator,
+ wserver->siridb->received_points,
+ cond->int64);
+
+ case CLERI_GID_K_SELECTED_POINTS:
+ return cexpr_int_cmp(
+ cond->operator,
+ wserver->siridb->selected_points,
+ cond->int64);
+
+ case CLERI_GID_K_UPTIME:
+ return cexpr_int_cmp(
+ cond->operator,
+ (int64_t) siridb_get_uptime(wserver->siridb),
+ cond->int64);
+
+ case CLERI_GID_K_ACTIVE_HANDLES:
+ return cexpr_int_cmp(
+ cond->operator,
+ (int64_t) siri.loop->active_handles,
+ cond->int64);
+
+ case CLERI_GID_K_ACTIVE_TASKS:
+ return cexpr_int_cmp(
+ cond->operator,
+ (int64_t) wserver->siridb->tasks.active,
+ cond->int64);
+
+ case CLERI_GID_K_IDLE_PERCENTAGE:
+ return cexpr_int_cmp(
+ cond->operator,
+ (int64_t) siridb_get_idle_percentage(wserver->siridb),
+ cond->int64);
+
+ case CLERI_GID_K_IDLE_TIME:
+ return cexpr_int_cmp(
+ cond->operator,
+ (int64_t) wserver->siridb->tasks.idle_time,
+ cond->int64);
+
+ case CLERI_GID_K_REINDEX_PROGRESS:
+ return cexpr_str_cmp(
+ cond->operator,
+ siridb_reindex_progress(wserver->siridb),
+ cond->str);
+
+ case CLERI_GID_K_SYNC_PROGRESS:
+ return cexpr_str_cmp(
+ cond->operator,
+ siridb_initsync_sync_progress(wserver->siridb),
+ cond->str);
+
+ case CLERI_GID_K_TEE_PIPE_NAME:
+ return cexpr_str_cmp(
+ cond->operator,
+ tee_str(wserver->siridb->tee),
+ cond->str);
+ }
+
+ log_critical("Unexpected server property received: %d", cond->prop);
+ assert (0);
+ return -1;
+}
+
+/*
+ * Cancel promise. The promise->cb will be called.
+ */
+static void SERVER_cancel_promise(sirinet_promise_t * promise)
+{
+ if (!uv_is_closing((uv_handle_t *) promise->timer))
+ {
+ uv_timer_stop(promise->timer);
+ uv_close((uv_handle_t *) promise->timer, (uv_close_cb) free);
+ }
+ promise->cb(promise, NULL, PROMISE_CANCELLED_ERROR);
+}
+
+
+/*
+ * Returns 0 if successful or -1 in case of an allocation error.
+ * (server->name is unchanged in case of an error)
+ */
+static int SERVER_update_name(siridb_server_t * server)
+{
+ /* start len with 2, one for : and one for 0 terminator */
+ size_t len = 2;
+ uint16_t i = server->port;
+ char * tmp;
+ int fmt_as_ipv6 = 0; /* false */
+
+ assert (server->port > 0);
+
+ /* append 'string' length for server->port */
+ for (; i; i /= 10, len++);
+
+ if FMT_AS_IPV6(server->address)
+ {
+ len += 2;
+ fmt_as_ipv6 = 1; /* true */
+ }
+
+ /* append 'address' length */
+ len += strlen(server->address);
+
+ assert (len > 0);
+
+ /* allocate enough space */
+ tmp = (char *) realloc(server->name, len);
+ if (tmp == NULL)
+ {
+ return -1;
+ }
+
+ server->name = tmp;
+
+ if (fmt_as_ipv6)
+ {
+ sprintf(server->name, "[%s]:%d", server->address, server->port);
+ }
+ else
+ {
+ sprintf(server->name, "%s:%d", server->address, server->port);
+ }
+ return 0;
+}
+
+static void SERVER_on_auth_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ if (status)
+ {
+ /* we already have a log entry so this can be a debug log */
+ log_debug(
+ "Error while sending authentication request to '%s' (%s)",
+ promise->server->name,
+ sirinet_promise_strstatus(status));
+ }
+ else if (pkg->tp == BPROTO_AUTH_SUCCESS)
+ {
+ log_info("Successful authenticated to server '%s'",
+ promise->server->name);
+
+ promise->server->flags |= SERVER_FLAG_AUTHENTICATED;
+ }
+ else
+ {
+ log_error("Authentication with server '%s' failed, error code: %d",
+ promise->server->name,
+ pkg->tp);
+ }
+
+ if ( (status || pkg->tp != BPROTO_AUTH_SUCCESS) &&
+ promise->server->client != NULL)
+ {
+ sirinet_stream_decref(promise->server->client);
+ }
+
+ /* we must free the promise */
+ sirinet_promise_decref(promise);
+}
+
+static void SERVER_on_flags_update_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ if (status)
+ {
+ /* we already have a log entry so this can be a debug log */
+ log_debug(
+ "Error while sending flags update to '%s' (%s)",
+ promise->server->name,
+ sirinet_promise_strstatus(status));
+
+ if (promise->server->client != NULL)
+ {
+ siridb_t * siridb = promise->server->client->siridb;
+ siridb_server_t * replica = siridb_servers_by_replica(
+ siridb->servers,
+ promise->server);
+ if (replica != NULL && (
+ replica == siridb->server ||
+ siridb_server_is_accessible(replica)))
+ {
+ /* we only set the status unavailable if we have an accessible
+ * replica or when we are the replica
+ */
+ log_warning(
+ "Set status for server '%s' to unavailable. "
+ "(unavailable status will be removed after a new status "
+ "is received)", promise->server->name);
+ promise->server->flags |= SERVER_FLAG_UNAVAILABLE;
+ }
+ }
+ }
+ else if (pkg->tp == BPROTO_ACK_FLAGS)
+ {
+ log_debug("Flags ACK received from '%s'", promise->server->name);
+ }
+ else
+ {
+ log_critical("Unexpected package type received from '%s' (type: %u)",
+ promise->server->name,
+ pkg->tp);
+ }
+
+ /* we must free the promise */
+ sirinet_promise_decref(promise);
+}
--- /dev/null
+/*
+ * servers.c - Collection of SiriDB servers.
+ */
+#include <assert.h>
+#include <llist/llist.h>
+#include <logger/logger.h>
+#include <procinfo/procinfo.h>
+#include <qpack/qpack.h>
+#include <siri/db/db.h>
+#include <siri/db/query.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/db/misc.h>
+#include <siri/db/tee.h>
+#include <siri/err.h>
+#include <siri/net/promises.h>
+#include <siri/net/tcp.h>
+#include <siri/db/queries.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <xpath/xpath.h>
+
+#define SIRIDB_SERVERS_FN "servers.dat"
+#define SIRIDB_SERVERS_SCHEMA 1
+
+static int SERVERS_walk_free(siridb_server_t * server, void * args);
+static int SERVERS_walk_save(siridb_server_t * server, qp_fpacker_t * fpacker);
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * (a signal can be raised in case of errors)
+ */
+int siridb_servers_load(siridb_t * siridb)
+{
+ log_info("Loading servers");
+
+ qp_unpacker_t * unpacker;
+ qp_obj_t qp_uuid;
+ qp_obj_t qp_address;
+ qp_obj_t qp_port;
+ qp_obj_t qp_pool;
+ siridb_server_t * server;
+ qp_types_t tp;
+
+ /* we should not have any servers at this moment */
+ assert(siridb->servers == NULL);
+
+ /* create a new server list */
+ siridb->servers = llist_new();
+ if (siridb->servers == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ /* get servers file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_SERVERS_FN)
+
+ if (!xpath_file_exist(fn))
+ {
+ /* we do not have a servers file, lets create the first server */
+
+ server = siridb_server_new(
+ (char *) siridb->uuid,
+ siri.cfg->server_address,
+ strlen(siri.cfg->server_address),
+ siri.cfg->listen_backend_port,
+ 0);
+ if (server == NULL)
+ {
+ return -1; /* signal is raised */
+ }
+
+ siridb_server_incref(server);
+ if (llist_append(siridb->servers, server))
+ {
+ siridb_server_decref(server);
+ ERR_ALLOC
+ return -1;
+ }
+
+ siridb->server = server;
+
+ if (siridb_servers_save(siridb))
+ {
+ log_critical("Cannot save servers");
+ return -1; /* signal is raised */
+ }
+
+ return 0;
+ }
+
+ if ((unpacker = qp_unpacker_ff(fn)) == NULL)
+ {
+ return -1; /* a signal is raised in case of memory allocation errors */
+ }
+
+ /* unpacker will be freed in case macro fails */
+ siridb_misc_schema_check(SIRIDB_SERVERS_SCHEMA)
+
+
+ int rc = 0;
+
+ while (qp_is_array(qp_next(unpacker, NULL)) &&
+ qp_next(unpacker, &qp_uuid) == QP_RAW &&
+ qp_uuid.len == 16 &&
+ qp_next(unpacker, &qp_address) == QP_RAW &&
+ qp_next(unpacker, &qp_port) == QP_INT64 &&
+ qp_next(unpacker, &qp_pool) == QP_INT64)
+ {
+ server = siridb_server_new(
+ (const char *) qp_uuid.via.raw,
+ (const char *) qp_address.via.raw,
+ qp_address.len,
+ (uint16_t) qp_port.via.int64,
+ (uint16_t) qp_pool.via.int64);
+ if (server == NULL)
+ {
+ rc = -1; /* signal is raised */
+ }
+ else
+ {
+ siridb_server_incref(server);
+
+ /* append the server to the list */
+ if (llist_append(siridb->servers, server))
+ {
+ siridb_server_decref(server);
+ rc = -1;
+ }
+ else if (uuid_compare(server->uuid, siridb->uuid) == 0)
+ {
+ /* if this is me, bind server to siridb->server */
+ siridb->server = server;
+ }
+ else
+ {
+ /* if this is not me, create promises */
+ server->promises = imap_new();
+ if (server->promises == NULL)
+ {
+ log_critical("Memory allocation error");
+ rc = -1;
+ }
+ }
+ }
+ }
+
+ /* save last object, should be QP_END */
+ tp = qp_next(unpacker, NULL);
+
+ /* free unpacker */
+ qp_unpacker_ff_free(unpacker);
+
+ if (siridb->server == NULL)
+ {
+ log_critical("Could not find my own uuid in '%s'", SIRIDB_SERVERS_FN);
+ rc = -1;
+ }
+ else if (tp != QP_END)
+ {
+ log_critical("Expected end of file '%s'", fn);
+ rc = -1;
+ }
+ else if (siridb_server_update_address(
+ siridb,
+ siridb->server,
+ siri.cfg->server_address,
+ siri.cfg->listen_backend_port) < 0)
+ {
+ rc = -1; /* logging is done, set result code to -1 */
+ }
+
+ return rc;
+}
+
+
+
+/*
+ * Destroy servers, parsing NULL is not allowed.
+ */
+void siridb_servers_free(llist_t * servers)
+{
+ llist_free_cb(servers, (llist_cb) SERVERS_walk_free, NULL);
+}
+
+/*
+ * Typedef: sirinet_clserver_get_file
+ *
+ * Returns the length of the content for a file and set buffer with the file
+ * content. Note that malloc is used to allocate memory for the buffer.
+ *
+ * In case of an error -1 is returned and buffer will be set to NULL.
+ */
+ssize_t siridb_servers_get_file(char ** buffer, siridb_t * siridb)
+{
+ /* get servers file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_SERVERS_FN)
+
+ return xpath_get_content(buffer, fn);
+}
+
+/*
+ * Returns 0 and increments server->ref by one if successful.
+ *
+ * In case of an error -1 is returned. (and a SIGNAL might be raised if
+ * this is a critical error)
+ */
+int siridb_servers_register(siridb_t * siridb, siridb_server_t * server)
+{
+ siridb_pool_t * pool;
+
+ if (server->pool < siridb->pools->len)
+ {
+ log_info("Register '%s' as a replica server for pool %u",
+ server->name,
+ server->pool);
+
+ pool = siridb->pools->pool + server->pool;
+ if (pool->len != 1)
+ {
+ log_error("Cannot register '%s' since pool %d contains %d servers",
+ server->name,
+ server->pool,
+ pool->len);
+ return -1;
+ }
+
+ if (siridb->server->pool == server->pool)
+ {
+ /* this is a replica for 'this' pool */
+ assert (siridb->replicate == NULL);
+ assert (siridb->fifo == NULL);
+ assert (siridb->replica == NULL);
+ siridb->replica = server;
+ siridb->fifo = siridb_fifo_new(siridb);
+
+ if (siridb->fifo == NULL)
+ {
+ log_critical(
+ "Cannot initialize fifo buffer for replica server");
+ /* signal is set */
+ return -1;
+ }
+
+ siridb_initsync_t * initsync = siridb_initsync_open(siridb, 1);
+ if (initsync == NULL && siridb->series_map->len)
+ {
+ log_critical(
+ "Cannot register '%s' because creating initial "
+ "synchronization file has failed",
+ server->name);
+ return -1;
+ }
+
+ if (siridb_replicate_init(siridb, initsync))
+ {
+ log_critical(
+ "Cannot register '%s' because replicate task "
+ "cannot be initialized",
+ server->name);
+ return -1;
+ }
+ }
+
+ siridb_pool_add_server(pool, server);
+ }
+ else if (server->pool > siridb->pools->len)
+ {
+ log_error("Cannot register '%s' since pool %d is unexpected",
+ server->name,
+ server->pool);
+ return -1;
+ }
+ else
+ {
+ log_info("Register '%s' for a new pool with id %u",
+ server->name,
+ server->pool);
+
+ pool = siridb_pools_append(siridb->pools, server);
+ if (pool == NULL)
+ {
+ log_critical(
+ "Cannot register '%s' because creating a new pool "
+ "has failed",
+ server->name);
+ return -1;
+ }
+
+ /* this is a new server for a new pool */
+ siridb->reindex = siridb_reindex_open(siridb, 1);
+ }
+
+ if (llist_append(siridb->servers, server) || siridb_servers_save(siridb))
+ {
+ log_critical("Cannot save server '%s'", server->name);
+ ERR_ALLOC
+ return -1;
+ }
+
+ siridb_server_incref(server);
+
+ /*
+ * Force one heart-beat to connect to the new server and
+ * sending the updated flags to the other servers.
+ */
+ siri_heartbeat_force();
+
+ if (siridb->reindex != NULL)
+ {
+ siridb_reindex_start(siridb->reindex->timer);
+ }
+
+ return 0;
+
+}
+
+/*
+ * Return a llist_t object containing all servers except 'this' server.
+ *
+ * Note: this function does not increase the servers reference counter.
+ *
+ * In case of an error, NULL is returned.
+ */
+vec_t * siridb_servers_other2vec(siridb_t * siridb)
+{
+ siridb_server_t * server;
+ vec_t * servers = vec_new(siridb->servers->len - 1);
+ llist_node_t * node;
+
+ if (servers == NULL)
+ {
+ return NULL;
+ }
+
+ for ( node = siridb->servers->first;
+ node != NULL;
+ node = node->next)
+ {
+ server = node->data;
+ if (server != siridb->server)
+ {
+ vec_append(servers, server);
+ }
+ }
+
+ return servers;
+}
+
+/*
+ * This function sends a package to all online server.
+ *
+ * Note:a signal can be raised but the callback will always be called.
+ *
+ * If promises could not be created, the 'cb' function will still be called
+ * using the arguments (NULL, data).
+ */
+void siridb_servers_send_pkg(
+ vec_t * servers,
+ sirinet_pkg_t * pkg,
+ uint64_t timeout,
+ sirinet_promises_cb cb,
+ void * data)
+{
+ sirinet_promises_t * promises =
+ sirinet_promises_new(servers->len, cb, data, pkg);
+ if (promises == NULL)
+ {
+ free(pkg);
+ cb(NULL, data);
+ }
+ else
+ {
+ sirinet_pkg_t * dup;
+ siridb_server_t * server;
+ size_t i;
+
+ for (i = 0; i < servers->len; i++)
+ {
+ server = (siridb_server_t *) servers->data[i];
+
+ if (siridb_server_is_online(server))
+ {
+ if ((dup = sirinet_pkg_dup(pkg)) == NULL ||
+ siridb_server_send_pkg(
+ server,
+ dup,
+ timeout,
+ (sirinet_promise_cb) sirinet_promises_on_response,
+ promises,
+ 0 /* flags */))
+ {
+ log_critical(
+ "Allocation error while trying to send a package "
+ "to '%s'", server->name);
+ free(dup);
+ vec_append(promises->promises, NULL);
+ }
+ }
+ else
+ {
+ log_debug("Cannot send package to '%s' (server is offline)",
+ server->name);
+ vec_append(promises->promises, NULL);
+ }
+
+ }
+ SIRINET_PROMISES_CHECK(promises)
+ }
+}
+
+siridb_server_t * siridb_servers_by_uuid(llist_t * servers, uuid_t uuid)
+{
+ llist_node_t * node = servers->first;
+ siridb_server_t * server;
+
+ while (node != NULL)
+ {
+ server = (siridb_server_t *) node->data;
+ if (uuid_compare(server->uuid, uuid) == 0)
+ {
+ return server;
+ }
+ node = node->next;
+ }
+ return NULL;
+}
+
+siridb_server_t * siridb_servers_by_name(llist_t * servers, const char * name)
+{
+ llist_node_t * node = servers->first;
+ siridb_server_t * server;
+
+ while (node != NULL)
+ {
+ server = (siridb_server_t *) node->data;
+ if (strcmp(server->name, name) == 0)
+ {
+ return server;
+ }
+ node = node->next;
+ }
+ return NULL;
+}
+
+siridb_server_t * siridb_servers_by_replica(
+ llist_t * servers,
+ siridb_server_t * replica)
+{
+
+ llist_node_t * node = servers->first;
+ siridb_server_t * server;
+ while (node != NULL)
+ {
+ server = (siridb_server_t *) node->data;
+ if (server != replica && server->pool == replica->pool)
+ {
+ return server;
+ }
+ node = node->next;
+ }
+ return NULL;
+}
+
+/*
+ * This function can raise a SIGNAL.
+ *
+ * Use this to send the current server->flags to all servers.
+ */
+void siridb_servers_send_flags(llist_t * servers)
+{
+ llist_node_t * node = servers->first;
+ siridb_server_t * server;
+
+ while (node != NULL)
+ {
+ server = node->data;
+ /*
+ * The AUTHENTICATED flag is never set on 'this' server and therefore
+ * 'this' server will be skipped as should.
+ */
+ if (siridb_server_is_online(server))
+ {
+ siridb_server_send_flags(server);
+ }
+ node = node->next;
+ }
+}
+
+/*
+ * Returns 1 (true) if all servers are online, 0 (false)
+ * if at least one server is not online. ('this' server is NOT included)
+ *
+ * Server is 'online' when at least running and authenticated but not
+ * queue-full
+ */
+int siridb_servers_online(siridb_t * siridb)
+{
+ llist_node_t * node = siridb->servers->first;
+ siridb_server_t * server;
+ while (node != NULL)
+ {
+ server = (siridb_server_t *) node->data;
+ if (siridb->server != server && !siridb_server_is_online(server))
+ {
+ return 0;
+ }
+ node = node->next;
+ }
+ return 1;
+}
+
+/*
+ * Returns 1 (true) if all servers are available, 0 (false)
+ * if at least one server is not available. ('this' server is NOT included)
+ *
+ * A server is 'available' when and ONLY when connected and authenticated.
+ */
+int siridb_servers_available(siridb_t * siridb)
+{
+ llist_node_t * node = siridb->servers->first;
+ siridb_server_t * server;
+ while (node != NULL)
+ {
+ server = (siridb_server_t *) node->data;
+ if (siridb->server != server && !siridb_server_is_available(server))
+ {
+ return 0;
+ }
+ node = node->next;
+ }
+ return 1;
+}
+
+int siridb_servers_list(siridb_server_t * server, uv_async_t * handle)
+{
+ siridb_query_t * query = handle->data;
+ query_list_t * qlist = query->data;
+ vec_t * props = qlist->props;
+ siridb_t * siridb = query->client->siridb;
+ cexpr_t * where_expr = qlist->where_expr;
+ size_t i;
+
+ siridb_server_walker_t wserver = {
+ .server=server,
+ .siridb=siridb
+ };
+
+ if (where_expr != NULL && !cexpr_run(
+ where_expr,
+ (cexpr_cb_t) siridb_server_cexpr_cb,
+ &wserver))
+ {
+ return 0; /* false */
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_OPEN);
+
+ for (i = 0; i < props->len; i++)
+ {
+ switch(*((uint32_t *) props->data[i]))
+ {
+ case CLERI_GID_K_ADDRESS:
+ qp_add_string(query->packer, server->address);
+ break;
+ case CLERI_GID_K_BUFFER_PATH:
+ qp_add_string(
+ query->packer,
+ (siridb->server == server) ?
+ siridb->buffer->path :
+ (server->buffer_path != NULL) ?
+ server->buffer_path : "");
+ break;
+ case CLERI_GID_K_BUFFER_SIZE:
+ qp_add_int64(
+ query->packer,
+ (siridb->server == server) ?
+ siridb->buffer->size : server->buffer_size);
+ break;
+ case CLERI_GID_K_DBPATH:
+ qp_add_string(
+ query->packer,
+ (siridb->server == server) ?
+ siridb->dbpath :
+ (server->dbpath != NULL) ?
+ server->dbpath : "");
+ break;
+ case CLERI_GID_K_IP_SUPPORT:
+ qp_add_string(
+ query->packer,
+ sirinet_tcp_ip_support_str((siridb->server == server) ?
+ siri.cfg->ip_support : server->ip_support));
+ break;
+ case CLERI_GID_K_LIBUV:
+ qp_add_string(
+ query->packer,
+ (siridb->server == server) ?
+ uv_version_string() :
+ (server->libuv != NULL) ?
+ server->libuv : "");
+ break;
+ case CLERI_GID_K_NAME:
+ qp_add_string(query->packer, server->name);
+ break;
+ case CLERI_GID_K_ONLINE:
+ qp_add_type(
+ query->packer,
+ (siridb->server == server || server->client != NULL) ?
+ QP_TRUE : QP_FALSE);
+ break;
+ case CLERI_GID_K_POOL:
+ qp_add_int64(query->packer, (int64_t) server->pool);
+ break;
+ case CLERI_GID_K_PORT:
+ qp_add_int64(query->packer, (int64_t) server->port);
+ break;
+ case CLERI_GID_K_STARTUP_TIME:
+ qp_add_int64(
+ query->packer,
+ (siridb->server == server) ?
+ siri.startup_time : server->startup_time);
+ break;
+ case CLERI_GID_K_STATUS:
+ {
+ char * status = siridb_server_str_status(server);
+ qp_add_string(query->packer, status);
+ free(status);
+ }
+
+ break;
+ case CLERI_GID_K_UUID:
+ {
+ char uuid[37];
+ uuid_unparse_lower(server->uuid, uuid);
+ qp_add_string(query->packer, uuid);
+ }
+ break;
+ case CLERI_GID_K_VERSION:
+ qp_add_string(
+ query->packer,
+ (siridb->server == server) ?
+ SIRIDB_VERSION :
+ (server->version != NULL) ?
+ server->version : "");
+ break;
+ /* all properties below are 'remote properties'. if a remote property
+ * is detected we should perform the query on each server and only for
+ * that specific server.
+ */
+ case CLERI_GID_K_ACTIVE_HANDLES:
+ qp_add_int64(
+ query->packer,
+ (int64_t) siri.loop->active_handles);
+ break;
+ case CLERI_GID_K_ACTIVE_TASKS:
+ qp_add_int64(
+ query->packer,
+ (int64_t) siridb->tasks.active);
+ break;
+ case CLERI_GID_K_IDLE_PERCENTAGE:
+ qp_add_int64(
+ query->packer,
+ siridb_get_idle_percentage(siridb));
+ break;
+ case CLERI_GID_K_IDLE_TIME:
+ qp_add_int64(
+ query->packer,
+ (int64_t) siridb->tasks.idle_time);
+ break;
+ case CLERI_GID_K_LOG_LEVEL:
+ qp_add_string(query->packer, Logger.level_name);
+ break;
+ case CLERI_GID_K_MAX_OPEN_FILES:
+ qp_add_int64(
+ query->packer,
+ (int64_t) siri.cfg->max_open_files);
+ break;
+ case CLERI_GID_K_MEM_USAGE:
+ qp_add_int64(
+ query->packer,
+ (int64_t) (procinfo_total_physical_memory() / 1024));
+ break;
+ case CLERI_GID_K_FIFO_FILES:
+ qp_add_int64(
+ query->packer,
+ (int64_t) siridb_fifo_size(siridb->fifo));
+ break;
+ case CLERI_GID_K_OPEN_FILES:
+ qp_add_int64(query->packer, siridb_open_files(siridb));
+ break;
+ case CLERI_GID_K_RECEIVED_POINTS:
+ qp_add_int64(query->packer, siridb->received_points);
+ break;
+ case CLERI_GID_K_REINDEX_PROGRESS:
+ qp_add_string(query->packer, siridb_reindex_progress(siridb));
+ break;
+ case CLERI_GID_K_SELECTED_POINTS:
+ qp_add_int64(query->packer, siridb->selected_points);
+ break;
+ case CLERI_GID_K_SYNC_PROGRESS:
+ qp_add_string(query->packer, siridb_initsync_sync_progress(siridb));
+ break;
+ case CLERI_GID_K_TEE_PIPE_NAME:
+ qp_add_string(query->packer, tee_str(siridb->tee));
+ break;
+ case CLERI_GID_K_UPTIME:
+ qp_add_int64(
+ query->packer,
+ siridb_get_uptime(siridb));
+ break;
+ }
+ }
+
+ qp_add_type(query->packer, QP_ARRAY_CLOSE);
+
+ return 1; /* true */
+}
+
+/*
+ * Returns the numbers of servers with a version less than the given version.
+ * If all servers are at least running the given version 0 is returned.
+ *
+ * Note: *this* server is not checked and is expected to have at least the
+ * required version.
+ */
+int siridb_servers_check_version(siridb_t * siridb, char * version)
+{
+ siridb_server_t * server;
+ int n = 0;
+ llist_node_t * node;
+
+ for ( node = siridb->servers->first;
+ node != NULL;
+ node = node->next)
+ {
+ server = (siridb_server_t *) node->data;
+ if (server != siridb->server)
+ {
+ n += (server->version == NULL ||
+ siri_version_cmp(server->version, version) < 0) ? 1 : 0;
+ }
+ }
+ return n;
+}
+
+/*
+ * Return 0 if successful or -1 and a SIGNAL is raised in case of an error.
+ */
+int siridb_servers_save(siridb_t * siridb)
+{
+ qp_fpacker_t * fpacker;
+
+ /* get servers file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_SERVERS_FN)
+
+ if ((fpacker = qp_open(fn, "w")) == NULL)
+ {
+ ERR_FILE
+ return -1;
+ }
+
+ if (
+ /* open a new array */
+ qp_fadd_type(fpacker, QP_ARRAY_OPEN) ||
+
+ /* write the current schema */
+ qp_fadd_int64(fpacker, SIRIDB_SERVERS_SCHEMA))
+ {
+ ERR_FILE
+ qp_close(fpacker);
+ return -1;
+ }
+
+ if (llist_walk(siridb->servers, (llist_cb) SERVERS_walk_save, fpacker))
+ {
+ ERR_FILE
+ qp_close(fpacker);
+ return -1;
+ }
+
+ /* close file pointer */
+ if (qp_close(fpacker))
+ {
+ ERR_FILE
+ return -1;
+ }
+
+ return 0;
+}
+
+static int SERVERS_walk_free(
+ siridb_server_t * server,
+ void * args __attribute__((unused)))
+{
+ siridb_server_decref(server);
+ return 0;
+}
+
+static int SERVERS_walk_save(siridb_server_t * server, qp_fpacker_t * fpacker)
+{
+ int rc = 0;
+ rc += qp_fadd_type(fpacker, QP_ARRAY4);
+ rc += qp_fadd_raw(fpacker, (unsigned char *) &server->uuid[0], 16);
+ rc += qp_fadd_string(fpacker, server->address);
+ rc += qp_fadd_int64(fpacker, (int64_t) server->port);
+ rc += qp_fadd_int64(fpacker, (int64_t) server->pool);
+ return rc;
+}
+
--- /dev/null
+/*
+ * shard.c - SiriDB shard file.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <ctree/ctree.h>
+#include <imap/imap.h>
+#include <limits.h>
+#include <logger/logger.h>
+#include <siri/db/series.h>
+#include <siri/db/shard.h>
+#include <siri/db/shards.h>
+#include <siri/db/points.h>
+#include <siri/optimize.h>
+#include <siri/err.h>
+#include <siri/file/pointer.h>
+#include <siri/siri.h>
+#include <vec/vec.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <xstr/xstr.h>
+
+/* max read buffer size used for reading from index file */
+#define SIRIDB_SHARD_MAX_CHUNK_SZ 65536
+
+/* growing with this block size */
+#define SHARD_GROW_SZ 131072
+
+/* shard schema (schemas below 20 are reserved for Python SiriDB) */
+#define SIRIDB_SHARD_SHEMA 21
+
+/* optimal points in a single shard */
+#define OPTIMAL_POINTS_PER_SHARD 2000
+
+/*
+ * Header schema layout
+ *
+ * Total Size 20
+ * 0 (uint8_t) SHEMA
+ * 1 (uint64_t) ID
+ * 9 (uint64_t) DURATION
+ * 17 (uint16_t) MAX_CHUCK_SZ
+ * 19 (uint8_t) TP
+ * 20 (uint8_t) TIME_PRECISION
+ * 21 (uint8_t) FLAGS
+ *
+ */
+#define HEADER_SIZE 22
+#define HEADER_SCHEMA 0
+#define HEADER_ID 1
+#define HEADER_DURATION 9
+#define HEADER_MAX_CHUNK_SZ 17
+#define HEADER_TP 19
+#define HEADER_TIME_PRECISION 20
+#define HEADER_FLAGS 21
+
+/* 0 (uint32_t) SERIES_ID
+ * 4 (uint32_t) START_TS
+ * 8 (uint32_t) END_TS
+ * 12 (uint16_t) LEN
+ * 14 (uint16_t) (OPTIONAL COMPRESSION INFO)
+ */
+#define IDX32_SZ 14 /* or 16 when log/compressed */
+#define IDX32E_SZ 16
+
+/* 0 (uint32_t) SERIES_ID
+ * 4 (uint64_t) START_TS
+ * 12 (uint64_t) END_TS
+ * 20 (uint16_t) LEN
+ * 22 (uint16_t) (OPTIONAL COMPRESSION INFO)
+ */
+#define IDX64_SZ 22 /* or 24 when log/compressed */
+#define IDX64E_SZ 24
+
+#define SHARD_STATUS_SIZE 8
+
+/*
+ * Once a shard is created the chunk_size is saved (and after a restart loaded)
+ * from the shard. Its not possible to shrink the chunk size for an existing
+ * shard since we assume the index will not grow when optimizing. It is
+ * possible to set a larger chunk size for an existing shard.
+ *
+ * New shards can be created using a lower max_chunk size.
+ *
+ * Max 65535 since uint16_t is used to store this value
+ */
+#define DEFAULT_MAX_CHUNK_SZ_NUM 800
+#define DEFAULT_MAX_CHUNK_SZ_LOG 128
+
+static const siridb_shard_flags_repr_t flags_map[SHARD_STATUS_SIZE] = {
+ {.repr="indexed", .flag=SIRIDB_SHARD_HAS_INDEX},
+ {.repr="overlap", .flag=SIRIDB_SHARD_HAS_OVERLAP},
+ {.repr="new-values", .flag=SIRIDB_SHARD_HAS_NEW_VALUES},
+ {.repr="dropped-series", .flag=SIRIDB_SHARD_HAS_DROPPED_SERIES},
+ {.repr="dropped", .flag=SIRIDB_SHARD_IS_REMOVED},
+ {.repr="loading", .flag=SIRIDB_SHARD_IS_LOADING},
+ {.repr="corrupt", .flag=SIRIDB_SHARD_IS_CORRUPT},
+ {.repr="compressed", .flag=SIRIDB_SHARD_IS_COMPRESSED},
+};
+
+const char shard_type_map[2][7] = {
+ "number",
+ "log"
+};
+
+static ssize_t SHARD_apply_idx(
+ siridb_t * siridb,
+ siridb_shard_t * shard,
+ char * pt,
+ size_t pos,
+ int is_ts64);
+static int SHARD_get_idx(
+ siridb_t * siridb,
+ siridb_shard_t * shard,
+ int is_ts64);
+static int SHARD_load_idx(
+ siridb_t * siridb,
+ siridb_shard_t * shard,
+ FILE * fp,
+ int is_ts64);
+static inline int SHARD_init_fn(siridb_t * siridb, siridb_shard_t * shard);
+static int SHARD_grow(siridb_shard_t * shard);
+static size_t SHARD_write_header(
+ siridb_t * siridb,
+ siridb_series_t * series,
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ FILE * fp);
+static int SHARD_remove(siridb_shard_t * shard);
+
+uint64_t siridb_shard_duration_from_interval(siridb_t * siridb, uint64_t interval)
+{
+ uint64_t x, n, week, day, hour;
+
+ n = interval * OPTIMAL_POINTS_PER_SHARD;
+
+ if (n == siridb->duration_num)
+ {
+ return siridb->duration_num;
+ }
+
+ if (n == siridb->duration_log)
+ {
+ return siridb->duration_log;
+ }
+
+ week = 3600*24*7*siridb->time->factor;
+ x = n / week;
+ if (x)
+ {
+ return (x + 1) * week;
+ }
+
+ day = 3600*24*siridb->time->factor;
+ x = n / day;
+ if (x)
+ {
+ return (x + 1) * day;
+ }
+
+ hour = 3600*siridb->time->factor;
+ x = n / hour;
+ return (x + 1) * hour;
+}
+
+uint64_t siridb_shard_interval_from_duration(uint64_t duration)
+{
+ return duration / OPTIMAL_POINTS_PER_SHARD;;
+}
+
+int siridb_shard_migrate(
+ siridb_t * siridb,
+ uint64_t shard_id,
+ uint64_t * duration)
+{
+ FILE * fp;
+ char * fn, * new_fn;
+ int rc;
+ size_t n;
+ uint8_t schema, tp;
+ rc = asprintf(
+ &fn,
+ "%s%s%" PRIu64 ".sdb",
+ siridb->dbpath,
+ SIRIDB_SHARDS_PATH,
+ shard_id);
+ if (rc < 0)
+ {
+ log_error("Cannot create shard filename");
+ return -1;
+ }
+
+ if ((fp = fopen(fn, "r")) == NULL)
+ {
+ log_error("Cannot open (old) shard file for reading: '%s'", fn);
+ return -1;
+ }
+
+ char header[HEADER_SIZE];
+
+ if (fread(&header, HEADER_SIZE, 1, fp) != 1)
+ {
+ /* cannot read header from shard file,
+ * close file decrement reference shard and return -1
+ */
+ fclose(fp);
+ log_critical("Missing header in (old) shard file: '%s'", fn);
+ return -1;
+ }
+
+ schema = (uint8_t) header[HEADER_SCHEMA];
+ if (schema > SIRIDB_SHARD_SHEMA)
+ {
+ fclose(fp);
+ log_critical(
+ "Shard file '%s' has schema '%u' which is not supported with "
+ "this version of SiriDB.", fn, schema);
+ return -1;
+ }
+
+ tp = (uint8_t) header[HEADER_TP];
+ fclose(fp);
+
+ *duration = tp == SIRIDB_SHARD_TP_NUMBER
+ ? siridb->duration_num
+ : siridb->duration_log;
+
+ rc = asprintf(
+ &new_fn,
+ "%s%s%016"PRIX64"_%016"PRIX64".sdb",
+ siridb->dbpath,
+ SIRIDB_SHARDS_PATH,
+ shard_id,
+ *duration);
+ if (rc < 0)
+ {
+ log_error("Cannot create new shard file name");
+ free(fn);
+ free(new_fn);
+ return -1;
+ }
+
+ (void) rename(fn, new_fn);
+
+ n = strlen(fn);
+ fn[n-3] = 'i';
+ fn[n-1] = 'x';
+
+ n = strlen(new_fn);
+ new_fn[n-3] = 'i';
+ new_fn[n-1] = 'x';
+
+ (void) rename(fn, new_fn);
+
+ free(fn);
+ free(new_fn);
+
+ return 0;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * When an error occurs, a SIGNAL can be raised in some cases but not for sure.
+ */
+int siridb_shard_load(siridb_t * siridb, uint64_t id, uint64_t duration)
+{
+ int is_ts64;
+ FILE * fp;
+ off_t shard_sz;
+ siridb_shard_t * shard = malloc(sizeof(siridb_shard_t));
+ omap_t * shards;
+
+ if (shard == NULL)
+ {
+ ERR_ALLOC
+ return -1; /* signal is raised */
+ }
+ shard->fp = siri_fp_new();
+ if (shard->fp == NULL)
+ {
+ free(shard);
+ return -1; /* signal is raised */
+ }
+
+ shard->id = id;
+ shard->ref = 1;
+ shard->len = HEADER_SIZE;
+ shard->replacing = NULL;
+ shard->duration = duration;
+
+ if (SHARD_init_fn(siridb, shard) < 0)
+ {
+ ERR_ALLOC
+ siridb_shard_decref(shard);
+ return -1; /* signal is raised */
+ }
+
+ log_info("Loading shard %" PRIu64, id);
+
+ if ((fp = fopen(shard->fn, "r")) == NULL)
+ {
+ log_error("Cannot open shard file for reading: '%s'", shard->fn);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ if (fseeko(fp, 0, SEEK_END) ||
+ (shard_sz = ftello(fp)) < (off_t) shard->len ||
+ fseeko(fp, 0, SEEK_SET))
+ {
+ fclose(fp);
+ log_critical("Index and/or shard corrupt: '%s'", shard->fn);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ shard->size = (size_t) shard_sz;
+
+ char header[HEADER_SIZE];
+
+ if (fread(&header, HEADER_SIZE, 1, fp) != 1)
+ {
+ /* cannot read header from shard file,
+ * close file decrement reference shard and return -1
+ */
+ fclose(fp);
+ log_critical("Missing header in shard file: '%s'", shard->fn);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ uint8_t schema = (uint8_t) header[HEADER_SCHEMA];
+ if (schema > SIRIDB_SHARD_SHEMA)
+ {
+ fclose(fp);
+ log_critical(
+ "Shard file '%s' has schema '%u' which is not supported with "
+ "this version of SiriDB.", shard->fn, schema);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ /* set shard type, flags and max_chunk_sz */
+ shard->tp = (uint8_t) header[HEADER_TP];
+ shard->flags = (uint8_t) header[HEADER_FLAGS] | SIRIDB_SHARD_IS_LOADING;
+ shard->max_chunk_sz = *((uint16_t *) (header + HEADER_MAX_CHUNK_SZ));
+
+ siridb_timep_t time_precision = (uint8_t) header[HEADER_TIME_PRECISION];
+
+ if (siridb->time->precision != time_precision)
+ {
+ fclose(fp);
+ log_critical(
+ "Time precision from shard (%s) is not the same as "
+ "database (%c). Skip loading '%c'",
+ siridb_time_short_map(time_precision),
+ siridb_time_short_map(siridb->time->precision),
+ shard->fn);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ switch (shard->tp)
+ {
+ case SIRIDB_SHARD_TP_NUMBER:
+ case SIRIDB_SHARD_TP_LOG:
+ is_ts64 = time_precision > SIRIDB_TIME_SECONDS;
+
+ if (SHARD_get_idx(siridb, shard, is_ts64))
+ {
+ fclose(fp);
+ log_critical("Cannot read index for shard: '%s'", shard->fn);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ if (shard->size > shard->len)
+ {
+ if (fseeko(fp, (off_t) shard->len, SEEK_SET))
+ {
+ fclose(fp);
+ log_critical("Seek error in: '%s'", shard->fn);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ SHARD_load_idx(siridb, shard, fp, is_ts64);
+ }
+ break;
+
+ default:
+ fclose(fp);
+ log_critical("Unknown type shard file: '%s'", shard->fn);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ if (fclose(fp))
+ {
+ log_critical("Cannot close shard file: '%s'", shard->fn);
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ shards = imap_get(siridb->shards, id);
+ if (shards == NULL)
+ {
+ shards = omap_create();
+ if (shards == NULL || imap_set(siridb->shards, id, shards) == -1)
+ {
+ siridb_shard_decref(shard);
+ return -1;
+ }
+ }
+
+ if (omap_set(shards, duration, shard) == NULL)
+ {
+ siridb_shard_decref(shard);
+ return -1;
+ }
+
+ /* remove LOADING flag from shard status */
+ shard->flags &= ~SIRIDB_SHARD_IS_LOADING;
+
+ return 0;
+}
+
+/*
+ * Create a new shard file and return a siridb_shard_t object.
+ *
+ * In case of an error the return value is NULL and a SIGNAL is raised.
+ */
+siridb_shard_t * siridb_shard_create(
+ siridb_t * siridb,
+ omap_t * shards,
+ uint64_t id,
+ uint64_t duration,
+ uint8_t tp,
+ siridb_shard_t * replacing)
+{
+ siridb_shard_t * shard = malloc(sizeof(siridb_shard_t));
+ FILE * fp;
+
+ if (shard == NULL)
+ {
+ ERR_ALLOC
+ return NULL;
+ }
+ if ((shard->fp = siri_fp_new()) == NULL)
+ {
+ free(shard);
+ return NULL; /* signal is raised */
+ }
+ shard->id = id;
+ shard->ref = 1;
+ shard->tp = tp;
+ shard->replacing = replacing;
+ shard->len = shard->size = HEADER_SIZE;
+ shard->duration = duration;
+ shard->max_chunk_sz = (replacing == NULL) ?
+ (tp == SIRIDB_SHARD_TP_NUMBER ?
+ DEFAULT_MAX_CHUNK_SZ_NUM : DEFAULT_MAX_CHUNK_SZ_LOG) :
+ replacing->max_chunk_sz;
+
+ if (SHARD_init_fn(siridb, shard) < 0)
+ {
+ siridb_shard_decref(shard);
+ ERR_ALLOC
+ return NULL;
+ }
+
+ shard->flags =
+ siri.cfg->shard_compression ? SIRIDB_SHARD_IS_COMPRESSED : 0;
+
+ shard->flags |=
+ (replacing == NULL || siri_optimize_create_idx(shard->fn)) ?
+ SIRIDB_SHARD_OK : SIRIDB_SHARD_HAS_INDEX;
+
+ if ((fp = fopen(shard->fn, "w")) == NULL)
+ {
+ char buf[1024];
+ log_critical("Cannot create shard file: '%s' (%s)",
+ shard->fn, strerror_si(errno, buf, sizeof(buf)));
+ siridb_shard_decref(shard);
+ ERR_FILE
+ return NULL;
+ }
+
+ /* 0 (uint8_t) SHEMA
+ * 1 (uint64_t) ID
+ * 9 (uint64_t) DURATION
+ * 17 (uint16_t) MAX_CHUNK_SZ
+ * 19 (uint8_t) TP
+ * 20 (uint8_t) TIME_PRECISION
+ * 21 (uint8_t) FLAGS
+ */
+ if ( fputc(SIRIDB_SHARD_SHEMA, fp) == EOF ||
+ fwrite(&id, sizeof(uint64_t), 1, fp) != 1 ||
+ fwrite(&duration, sizeof(uint64_t), 1, fp) != 1 ||
+ fwrite(&shard->max_chunk_sz, sizeof(uint16_t), 1, fp) != 1 ||
+ fputc(tp, fp) == EOF ||
+ fputc(siridb->time->precision, fp) == EOF ||
+ fputc(shard->flags, fp) == EOF)
+ {
+ char buf[1024];
+ log_critical("Cannot write to shard file: '%s' (%s)",
+ shard->fn, strerror_si(errno, buf, sizeof(buf)));
+ fclose(fp);
+ siridb_shard_decref(shard);
+ ERR_FILE
+ return NULL;
+ }
+
+ if (fclose(fp))
+ {
+ char buf[1024];
+ log_critical("Cannot close shard file: '%s' (%s)",
+ shard->fn, strerror_si(errno, buf, sizeof(buf)));
+ siridb_shard_decref(shard);
+ ERR_FILE
+ return NULL;
+ }
+
+ if (omap_set(shards, duration, shard) == NULL)
+ {
+ siridb_shard_decref(shard);
+ ERR_ALLOC
+ return NULL;
+ }
+
+ /*
+ * This is not critical at this point and it's hard to imagine this
+ * fails if all the above was successful
+ */
+ siri_fopen(siri.fh, shard->fp, shard->fn, "r+");
+
+ return shard;
+}
+
+/*
+ * Call-back function used to validate shards in a where expression.
+ *
+ * Returns 0 or 1 (false or true).
+ */
+int siridb_shard_cexpr_cb(
+ siridb_shard_view_t * vshard,
+ cexpr_condition_t * cond)
+{
+ switch (cond->prop)
+ {
+ case CLERI_GID_K_SID:
+ return cexpr_int_cmp(cond->operator, vshard->shard->id, cond->int64);
+ case CLERI_GID_K_POOL:
+ return cexpr_int_cmp(cond->operator, vshard->server->pool, cond->int64);
+ case CLERI_GID_K_SIZE:
+ return cexpr_int_cmp(cond->operator, vshard->shard->len, cond->int64);
+ case CLERI_GID_K_START:
+ return cexpr_int_cmp(cond->operator, vshard->start, cond->int64);
+ case CLERI_GID_K_END:
+ return cexpr_int_cmp(cond->operator, vshard->end, cond->int64);
+ case CLERI_GID_K_TYPE:
+ return cexpr_int_cmp(cond->operator, vshard->shard->tp, cond->int64);
+ case CLERI_GID_K_SERVER:
+ return cexpr_str_cmp(cond->operator, vshard->server->name, cond->str);
+ case CLERI_GID_K_STATUS:
+ {
+ char buffer[SIRIDB_SHARD_STATUS_STR_MAX];
+ siridb_shard_status(buffer, vshard->shard);
+ return cexpr_str_cmp(cond->operator, buffer, cond->str);
+ }
+ }
+ log_critical("Unexpected shard property received: %d", cond->prop);
+ assert (0);
+ return -1;
+}
+
+/*
+ * Make sure 'str' is a pointer to a string which can hold at least
+ * SIRIDB_SHARD_STR_MAX.
+ */
+int siridb_shard_status(char * str, siridb_shard_t * shard)
+{
+ char * pt = str;
+ int i;
+ uint8_t flags;
+
+ if (shard->replacing != NULL)
+ {
+ pt += sprintf(pt, "optimizing");
+ }
+
+ flags = shard->flags;
+
+ for (i = 1; i < SHARD_STATUS_SIZE && flags; i++)
+ {
+ if ((flags & flags_map[i].flag) == flags_map[i].flag)
+ {
+ flags -= flags_map[i].flag;
+ pt += (pt == str) ? sprintf(pt, "%s", flags_map[i].repr) :
+ sprintf(pt, " | %s", flags_map[i].repr);
+ }
+ }
+
+ if (pt == str)
+ {
+ pt += sprintf(pt, "ok");
+ }
+ return pt - str;
+}
+
+/*
+ * Writes an index and points to a shard. The return value is the position
+ * where the points start in the shard file.
+ *
+ * If an error has occurred, 0 will be returned and a SIGNAL will be raised.
+ */
+size_t siridb_shard_write_points(
+ siridb_t * siridb,
+ siridb_series_t * series,
+ siridb_shard_t * shard,
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ FILE * idx_fp,
+ uint16_t * cinfo)
+{
+ FILE * fp;
+ uint16_t len = end - start;
+ size_t dsize;
+ unsigned char * cdata = NULL;
+
+ uint_fast32_t i;
+ size_t pos, header_sz;
+
+ if (shard->fp->fp == NULL)
+ {
+ if (siri_fopen(siri.fh, shard->fp, shard->fn, "r+"))
+ {
+ char buf[1024];
+ log_critical("Cannot open file '%s' (%s)",
+ shard->fn, strerror_r(errno, buf, 1024));
+ ERR_FILE
+ return 0;
+ }
+ }
+ fp = shard->fp->fp;
+
+ if (shard->flags & SIRIDB_SHARD_IS_COMPRESSED)
+ {
+ cdata = siridb_points_zip(points, start, end, cinfo, &dsize);
+ if (cdata == NULL)
+ {
+ ERR_ALLOC
+ log_critical("Memory allocation error while compressing points");
+ return 0;
+ }
+ }
+ else if (series->tp == TP_STRING)
+ {
+ dsize = siridb->time->ts_sz;
+ cdata = siridb_points_raw_string(points, start, end, cinfo, &dsize);
+ if (cdata == NULL)
+ {
+ ERR_ALLOC
+ log_critical("Memory allocation error while compressing points");
+ return 0;
+ }
+ }
+ else
+ {
+ /* no compression, ignore c-info */
+ cinfo = NULL;
+ dsize = (siridb->time->ts_sz + 8) * len;
+ }
+
+ if (shard->len > SHARD_GROW_SZ && (shard->len + dsize + 64 > shard->size))
+ {
+ SHARD_grow(shard);
+ }
+
+ if (fseeko(fp, shard->len, SEEK_SET))
+ {
+ log_critical("Seek error in: '%s'", shard->fn);
+ return 0;
+ }
+
+ if (idx_fp == NULL || (shard->flags & SIRIDB_SHARD_HAS_NEW_VALUES))
+ {
+ header_sz = SHARD_write_header(
+ siridb,
+ series,
+ points,
+ start,
+ end,
+ cinfo,
+ fp);
+ pos = shard->len + header_sz;
+ }
+ else
+ {
+ /* the idx_fp is already at the end of the index file */
+ header_sz = SHARD_write_header(
+ siridb,
+ series,
+ points,
+ start,
+ end,
+ cinfo,
+ idx_fp);
+ pos = shard->len;
+ }
+
+ if (!header_sz)
+ {
+ ERR_FILE
+ log_critical(
+ "Cannot write index header for shard id %" PRIu64,
+ shard->id);
+ free(cdata);
+ return 0;
+ }
+
+ if (cdata == NULL)
+ {
+ size_t p = 0;
+ size_t ts_sz = siridb->time->ts_sz;
+
+ cdata = malloc(dsize);
+ if (cdata == NULL)
+ {
+ ERR_ALLOC
+ log_critical("Memory allocation error while compressing points");
+ return 0;
+ }
+
+ for (i = start; i < end; i++)
+ {
+ memcpy(cdata + p, &points->data[i].ts, ts_sz);
+ p += ts_sz;
+ memcpy(cdata + p, &points->data[i].val, 8);
+ p += 8;
+ }
+ }
+
+ long int rc = fwrite(cdata, dsize, 1, fp);
+
+ if (rc != 1 || fflush(fp))
+ {
+ char buf[1024];
+ log_critical("Cannot write points to file '%s' (%s)",
+ shard->fn, strerror_r(errno, buf, 1024));
+ ERR_FILE
+ return 0;
+ }
+
+ free(cdata);
+
+ shard->len = pos + dsize;
+ return pos;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error. SiriDB might recover
+ * from this error so we do not consider this critical.
+ */
+int siridb_shard_get_points_num32(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ uint32_t * temp,* pt;
+ size_t len = points->len + idx->len;
+
+ if (idx->shard->fp->fp == NULL)
+ {
+ if (siri_fopen(siri.fh, idx->shard->fp, idx->shard->fn, "r+"))
+ {
+ log_critical(
+ "Cannot open file '%s', skip reading points",
+ idx->shard->fn);
+ return -1;
+ }
+ }
+
+ temp = malloc(sizeof(uint32_t) * idx->len * 3);
+ if (temp == NULL)
+ {
+ log_critical("Memory allocation error");
+ return -1;
+ }
+
+ if (fseeko(idx->shard->fp->fp, idx->pos, SEEK_SET) ||
+ fread(
+ temp,
+ 12, /* NUM32 point size */
+ idx->len,
+ idx->shard->fp->fp) != idx->len)
+ {
+ if (idx->shard->flags & SIRIDB_SHARD_IS_CORRUPT)
+ {
+ log_error("Cannot read from shard id %" PRIu64, idx->shard->id);
+ }
+ else
+ {
+ log_critical(
+ "Cannot read from shard id %" PRIu64
+ ". The next optimize cycle "
+ "will fix this shard but you might loose some data.",
+ idx->shard->id);
+ idx->shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ }
+ free(temp);
+ return -1;
+ }
+
+ /* set pointer to start */
+ pt = temp;
+
+ /* crop from start if needed */
+ if (start_ts != NULL)
+ {
+ for (; *pt < *start_ts; pt += 3, len--);
+ }
+
+ /* crop from end if needed */
+ if (end_ts != NULL)
+ {
+ uint32_t * p;
+ for ( p = temp + 3 * (idx->len - 1);
+ *p >= *end_ts;
+ p -= 3, len--);
+ }
+
+ if ( has_overlap &&
+ points->len &&
+ (idx->shard->flags & SIRIDB_SHARD_HAS_OVERLAP))
+ {
+ uint64_t ts;
+ for (; points->len < len; pt += 3)
+ {
+ ts = (uint64_t) *pt;
+ siridb_points_add_point(points, &ts, ((qp_via_t *) (pt + 1)));
+ }
+ }
+ else
+ {
+ for (; points->len < len; points->len++, pt += 3)
+ {
+ points->data[points->len].ts = (uint64_t) *pt;
+ points->data[points->len].val = *((qp_via_t *) (pt + 1));
+ }
+ }
+
+ free(temp);
+ return 0;
+}
+
+/*
+ * COPY from siridb_shard_get_points_num32
+ */
+int siridb_shard_get_points_num64(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ uint64_t * temp, * pt;
+ size_t len = points->len + idx->len;
+
+ if (idx->shard->fp->fp == NULL)
+ {
+ if (siri_fopen(siri.fh, idx->shard->fp, idx->shard->fn, "r+"))
+ {
+ log_critical(
+ "Cannot open file '%s', skip reading points",
+ idx->shard->fn);
+ return -1;
+ }
+ }
+
+ temp = malloc(sizeof(uint64_t) * idx->len * 2);
+ if (temp == NULL)
+ {
+ log_critical("Memory allocation error");
+ return -1;
+ }
+
+ if (fseeko(idx->shard->fp->fp, idx->pos, SEEK_SET) ||
+ fread(
+ temp,
+ 16, /* NUM64 point size */
+ idx->len,
+ idx->shard->fp->fp) != idx->len)
+ {
+ if (idx->shard->flags & SIRIDB_SHARD_IS_CORRUPT)
+ {
+ log_error("Cannot read from shard id %" PRIu64, idx->shard->id);
+ }
+ else
+ {
+ log_critical(
+ "Cannot read from shard id %" PRIu64
+ ". The next optimize cycle "
+ "will fix this shard but you might loose some data.",
+ idx->shard->id);
+ idx->shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ }
+ free(temp);
+ return -1;
+ }
+
+ /* set pointer to start */
+ pt = temp;
+
+ /* crop from start if needed */
+ if (start_ts != NULL)
+ {
+ for (; *pt < *start_ts; pt += 2, len--);
+ }
+
+ /* crop from end if needed */
+ if (end_ts != NULL)
+ {
+ uint64_t * p;
+ for ( p = temp + 2 * (idx->len - 1);
+ *p >= *end_ts;
+ p -= 2, len--);
+ }
+
+ if ( has_overlap &&
+ points->len &&
+ (idx->shard->flags & SIRIDB_SHARD_HAS_OVERLAP))
+ {
+ for (; points->len < len; pt += 2)
+ {
+ siridb_points_add_point(points, pt, ((qp_via_t *) (pt + 1)));
+ }
+ }
+ else
+ {
+ for (; points->len < len; points->len++, pt += 2)
+ {
+ points->data[points->len].ts = *pt;
+ points->data[points->len].val = *((qp_via_t *) (pt + 1));
+ }
+ }
+
+ free(temp);
+ return 0;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error. SiriDB might recover
+ * from this error so we do not consider this critical.
+ */
+int siridb_shard_get_points_num_compressed(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ unsigned char * bits;
+ size_t size = siridb_points_get_size_zipped(idx->cinfo, idx->len);
+
+ if (idx->shard->fp->fp == NULL)
+ {
+ if (siri_fopen(siri.fh, idx->shard->fp, idx->shard->fn, "r+"))
+ {
+ log_critical(
+ "Cannot open file '%s', skip reading points",
+ idx->shard->fn);
+ return -1;
+ }
+ }
+
+ bits = malloc(size);
+ if (bits == NULL)
+ {
+ log_critical("Memory allocation error");
+ return -1;
+ }
+
+ if (fseeko(idx->shard->fp->fp, idx->pos, SEEK_SET) ||
+ fread(bits, size, 1, idx->shard->fp->fp) != 1)
+ {
+ if (idx->shard->flags & SIRIDB_SHARD_IS_CORRUPT)
+ {
+ log_error("Cannot read from shard id %" PRIu64, idx->shard->id);
+ }
+ else
+ {
+ log_critical(
+ "Cannot read from shard id %" PRIu64
+ ". The next optimize cycle "
+ "will fix this shard but you might loose some data.",
+ idx->shard->id);
+ idx->shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ }
+ free(bits);
+ return -1;
+ }
+
+ switch (points->tp)
+ {
+ case TP_INT:
+ siridb_points_unzip_int(
+ points,
+ bits,
+ idx->len,
+ idx->cinfo,
+ start_ts,
+ end_ts,
+ has_overlap && (idx->shard->flags & SIRIDB_SHARD_HAS_OVERLAP));
+ break;
+ case TP_DOUBLE:
+ siridb_points_unzip_double(
+ points,
+ bits,
+ idx->len,
+ idx->cinfo,
+ start_ts,
+ end_ts,
+ has_overlap && (idx->shard->flags & SIRIDB_SHARD_HAS_OVERLAP));
+ break;
+ case TP_STRING: assert(0);
+ }
+
+ free(bits);
+ return 0;
+}
+
+int siridb_shard_get_points_log_compressed(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ int rc;
+
+ if (idx->len < POINTS_ZIP_THRESHOLD)
+ {
+ return siridb_shard_get_points_log64(
+ points, idx, start_ts, end_ts, has_overlap);
+ }
+
+ uint8_t * bits;
+ size_t size = siridb_points_get_size_log(idx->cinfo);
+
+ if (idx->shard->fp->fp == NULL)
+ {
+ if (siri_fopen(siri.fh, idx->shard->fp, idx->shard->fn, "r+"))
+ {
+ log_critical(
+ "Cannot open file '%s', skip reading points",
+ idx->shard->fn);
+ return -1;
+ }
+ }
+ bits = malloc(size);
+ if (bits == NULL)
+ {
+ free(bits);
+ log_critical("Memory allocation error");
+ return -1;
+ }
+
+ if ( fseeko(idx->shard->fp->fp, idx->pos, SEEK_SET) ||
+ fread( bits,
+ sizeof(uint8_t),
+ size,
+ idx->shard->fp->fp) != size)
+ {
+ if (idx->shard->flags & SIRIDB_SHARD_IS_CORRUPT)
+ {
+ log_error("Cannot read from shard id %" PRIu64, idx->shard->id);
+ }
+ else
+ {
+ log_critical(
+ "Cannot read from shard id %" PRIu64
+ ". The next optimize cycle "
+ "will fix this shard but you might loose some data.",
+ idx->shard->id);
+ idx->shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ }
+ free(bits);
+ return -1;
+ }
+
+ rc = siridb_points_unzip_string(
+ points,
+ bits,
+ idx->len,
+ start_ts,
+ end_ts,
+ has_overlap && (idx->shard->flags & SIRIDB_SHARD_HAS_OVERLAP));
+
+ free(bits);
+
+ return rc;
+}
+
+/*
+ * COPY from siridb_shard_get_points_log64
+ */
+int siridb_shard_get_points_log32(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ uint32_t * tdata, * tpt;
+ char * cdata, * cpt;
+ size_t len = points->len + idx->len;
+ size_t dsize = siridb_points_get_size_log(idx->cinfo);
+
+ if (idx->shard->fp->fp == NULL)
+ {
+ if (siri_fopen(siri.fh, idx->shard->fp, idx->shard->fn, "r+"))
+ {
+ log_critical(
+ "Cannot open file '%s', skip reading points",
+ idx->shard->fn);
+ return -1;
+ }
+ }
+
+ tdata = malloc(sizeof(uint32_t) * idx->len);
+ cdata = malloc(dsize);
+ if (cdata == NULL || tdata == NULL)
+ {
+ free(tdata);
+ free(cdata);
+ log_critical("Memory allocation error");
+ return -1;
+ }
+
+ if ( fseeko(idx->shard->fp->fp, idx->pos, SEEK_SET) ||
+ fread( tdata,
+ sizeof(uint32_t),
+ idx->len,
+ idx->shard->fp->fp) != idx->len ||
+ fread( cdata,
+ sizeof(unsigned char),
+ dsize,
+ idx->shard->fp->fp) != dsize)
+ {
+ if (idx->shard->flags & SIRIDB_SHARD_IS_CORRUPT)
+ {
+ log_error("Cannot read from shard id %" PRIu64, idx->shard->id);
+ }
+ else
+ {
+ log_critical(
+ "Cannot read from shard id %" PRIu64
+ ". The next optimize cycle "
+ "will fix this shard but you might loose some data.",
+ idx->shard->id);
+ idx->shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ }
+ free(tdata);
+ free(cdata);
+ return -1;
+ }
+
+ /* set pointer to start */
+ tpt = tdata;
+ cpt = cdata;
+
+ /* crop from start if needed */
+ if (start_ts != NULL)
+ {
+ for (; *tpt < *start_ts;)
+ {
+ tpt++;
+ for(; *cpt; ++cpt);
+ ++cpt;
+ len--;
+ }
+ }
+
+ /* crop from end if needed */
+ if (end_ts != NULL)
+ {
+ uint32_t * p;
+ for (p = tdata + (idx->len - 1); *p >= *end_ts; --p, len--);
+ }
+
+ if ( has_overlap &&
+ points->len &&
+ (idx->shard->flags & SIRIDB_SHARD_HAS_OVERLAP))
+ {
+ for (; points->len < len; ++tpt)
+ {
+ size_t slen;
+ qp_via_t v;
+ v.str = xstr_dup(cpt, &slen);
+ cpt += slen + 1;
+ uint64_t ts = *tpt;
+ siridb_points_add_point(points, &ts, &v);
+ }
+ }
+ else
+ {
+ for (; points->len < len; points->len++, ++tpt)
+ {
+ size_t slen;
+ points->data[points->len].ts = *tpt;
+ points->data[points->len].val.str = xstr_dup(cpt, &slen);
+ cpt += slen + 1;
+ }
+ }
+
+ free(tdata);
+ free(cdata);
+ return 0;
+}
+
+/*
+ * COPY from siridb_shard_get_points_log32
+ */
+int siridb_shard_get_points_log64(
+ siridb_points_t * points,
+ idx_t * idx,
+ uint64_t * start_ts,
+ uint64_t * end_ts,
+ uint8_t has_overlap)
+{
+ uint64_t * tdata, * tpt;
+ char * cdata, * cpt;
+ size_t len = points->len + idx->len;
+ size_t dsize = siridb_points_get_size_log(idx->cinfo);
+
+ if (idx->shard->fp->fp == NULL)
+ {
+ if (siri_fopen(siri.fh, idx->shard->fp, idx->shard->fn, "r+"))
+ {
+ log_critical(
+ "Cannot open file '%s', skip reading points",
+ idx->shard->fn);
+ return -1;
+ }
+ }
+
+ tdata = malloc(sizeof(uint64_t) * idx->len);
+ cdata = malloc(dsize);
+ if (cdata == NULL || tdata == NULL)
+ {
+ free(tdata);
+ free(cdata);
+ log_critical("Memory allocation error");
+ return -1;
+ }
+
+ if ( fseeko(idx->shard->fp->fp, idx->pos, SEEK_SET) ||
+ fread( tdata,
+ sizeof(uint64_t),
+ idx->len,
+ idx->shard->fp->fp) != idx->len ||
+ fread( cdata,
+ sizeof(unsigned char),
+ dsize,
+ idx->shard->fp->fp) != dsize)
+ {
+ if (idx->shard->flags & SIRIDB_SHARD_IS_CORRUPT)
+ {
+ log_error("Cannot read from shard id %" PRIu64, idx->shard->id);
+ }
+ else
+ {
+ log_critical(
+ "Cannot read from shard id %" PRIu64
+ ". The next optimize cycle "
+ "will fix this shard but you might loose some data.",
+ idx->shard->id);
+ idx->shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ }
+ free(tdata);
+ free(cdata);
+ return -1;
+ }
+
+ /* set pointer to start */
+ tpt = tdata;
+ cpt = cdata;
+
+ /* crop from start if needed */
+ if (start_ts != NULL)
+ {
+ for (; *tpt < *start_ts;)
+ {
+ tpt++;
+ for(; *cpt; ++cpt);
+ ++cpt;
+ len--;
+ }
+ }
+
+ /* crop from end if needed */
+ if (end_ts != NULL)
+ {
+ uint64_t * p;
+ for (p = tdata + (idx->len - 1); *p >= *end_ts; --p, len--);
+ }
+
+ if ( has_overlap &&
+ points->len &&
+ (idx->shard->flags & SIRIDB_SHARD_HAS_OVERLAP))
+ {
+ for (; points->len < len; ++tpt)
+ {
+ size_t slen;
+ qp_via_t v;
+ v.str = xstr_dup(cpt, &slen);
+ cpt += slen + 1;
+ siridb_points_add_point(points, tpt, &v);
+ }
+ }
+ else
+ {
+ for (; points->len < len; points->len++, ++tpt)
+ {
+ size_t slen;
+ points->data[points->len].ts = *tpt;
+ points->data[points->len].val.str = xstr_dup(cpt, &slen);
+ cpt += slen + 1;
+ }
+ }
+
+ free(tdata);
+ free(cdata);
+ return 0;
+}
+
+/*
+ * This function will be called from the 'optimize' thread.
+ *
+ * Returns 0 if successful or -1 and a SIGNAL is raised in case of an error.
+ */
+int siridb_shard_optimize(siridb_shard_t * shard, siridb_t * siridb)
+{
+ int rc = 0;
+ siridb_shard_t * new_shard = NULL;
+ siridb_series_t * series;
+ size_t i;
+
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ /* In case the shard is not removed, it must be the shard inside the imap
+ * because we check and replace the shard within the shards_mutex lock.
+ * If the shard is marked as removed we can simply skip the optimize.
+ */
+ if (~shard->flags & SIRIDB_SHARD_IS_REMOVED)
+ {
+ omap_t * shards = imap_get(siridb->shards, shard->id);
+ assert (shards);
+
+ if ((new_shard = siridb_shard_create(
+ siridb,
+ shards,
+ shard->id,
+ shard->duration,
+ shard->tp,
+ shard)) == NULL)
+ {
+ /* signal is raised */
+ log_critical(
+ "Cannot create shard id '%" PRIu64 "' for optimizing.",
+ shard->id);
+ rc = -1;
+ }
+ else
+ {
+ siridb_shard_incref(new_shard);
+ }
+ }
+ else
+ {
+ log_warning(
+ "Skip optimizing shard id '%" PRIu64 "' "
+ "because the shard is probably dropped.",
+ shard->id);
+ }
+
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ if (new_shard == NULL)
+ {
+ /*
+ * Creating the new shard has failed or the shard is dropped.
+ * We exit here so the mutex is is unlocked.
+ * (a signal might have been raised)
+ */
+ return rc;
+ }
+
+ /* at this point the references should be as following (unless dropped):
+ * shard->ref (=>2)
+ * - simple list
+ * - new_shard->replacing
+ * new_shard->ref (=>2)
+ * - siridb->shards
+ * - this method
+ */
+
+ sleep(1);
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ vec_t * vec = imap_2vec_ref(siridb->series_map);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ if (vec == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ sleep(1);
+
+ for (i = 0; i < vec->len; i++)
+ {
+ /* its possible that another database is paused, but we wait anyway */
+ if (siri.optimize->pause)
+ {
+ siri_optimize_wait();
+ }
+
+ series = vec->data[i];
+
+ if ( !siri_err &&
+ siri.optimize->status != SIRI_OPTIMIZE_CANCELLED &&
+ shard->id % shard->duration == series->mask &&
+ (~series->flags & SIRIDB_SERIES_IS_DROPPED) &&
+ (~new_shard->flags & SIRIDB_SHARD_IS_REMOVED))
+ {
+ uv_mutex_lock(&siridb->series_mutex);
+
+ if ( (~new_shard->flags & SIRIDB_SHARD_IS_REMOVED) &&
+ siridb_series_optimize_shard(
+ siridb,
+ series,
+ new_shard))
+ {
+ log_critical(
+ "Optimizing shard '%s' has failed due to a critical "
+ "error", shard->fn);
+ }
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ /* make this sleep depending on the active_tasks
+ * (50ms per active task) */
+ usleep( 50000 * siridb->tasks.active + 100 );
+ }
+
+ siridb_series_decref(series);
+ }
+
+ vec_free(vec);
+
+ if (new_shard->flags & SIRIDB_SHARD_IS_REMOVED)
+ {
+ log_warning(
+ "Cancel optimizing shard '%s' because the shard is dropped",
+ new_shard->fn);
+ siridb_shard_decref(new_shard);
+ return siri_err;
+ }
+
+ if (siri_err || siri.optimize->status == SIRI_OPTIMIZE_CANCELLED)
+ {
+ /*
+ * Error occurred or the optimize task is cancelled. By decrementing
+ * only the reference counter for the new_shard we keep this shard as
+ * if it is still optimizing so remaining points can still be written.
+ */
+ siridb_shard_decref(new_shard);
+ return siri_err;
+ }
+
+ sleep(1);
+
+ uv_mutex_lock(&siridb->series_mutex);
+
+ /* make sure both shards files are closed */
+ siri_fp_close(new_shard->replacing->fp);
+ siri_fp_close(new_shard->fp);
+
+ /*
+ * Closing files or writing to the new shard might have produced
+ * critical errors. This seems to be a good point to check for errors.
+ */
+ if (siri_err || (new_shard->flags & SIRIDB_SHARD_IS_REMOVED))
+ {
+ if (new_shard->flags & SIRIDB_SHARD_IS_REMOVED)
+ {
+ log_warning(
+ "Cancel optimizing shard '%s' because the shard is dropped",
+ new_shard->fn);
+ }
+ }
+ else
+ {
+ /* remove the old shard file, this is not critical */
+ unlink(new_shard->replacing->fn);
+
+ /* rename the temporary files to the correct file names */
+ if (rename(new_shard->fn, new_shard->replacing->fn) ||
+ siri_optimize_finish_idx(
+ new_shard->replacing->fn,
+ new_shard->replacing->flags & SIRIDB_SHARD_HAS_INDEX))
+ {
+ log_critical(
+ "Could not rename file '%s' to '%s'",
+ new_shard->fn,
+ new_shard->replacing->fn);
+ ERR_FILE
+ }
+ else
+ {
+ /* free the original allocated memory and set the new filename */
+ free(new_shard->fn);
+ new_shard->fn = new_shard->replacing->fn;
+ new_shard->replacing->fn = NULL;
+
+ /* decrement reference to old shard and set
+ * new_shard->replacing to NULL
+ */
+ siridb_shard_decref(new_shard->replacing);
+ new_shard->replacing = NULL;
+ }
+ }
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ /* can raise an error only if the shard is dropped, in any other case we
+ * still have a reference left and an error cannot be raised.
+ */
+ siridb_shard_decref(new_shard);
+
+ sleep(1);
+
+ return siri_err;
+}
+
+/*
+ * This function can be used instead of the macro function when needed as
+ * callback.
+ *
+ * Decrement the reference counter, when 0 the shard will be destroyed.
+ *
+ * In case the shard will be destroyed and flag SIRIDB_SHARD_WILL_BE_REMOVED
+ * is set, the file will be removed.
+ *
+ * A signal can be raised in case closing the shard file fails.
+ */
+void siridb__shard_decref(siridb_shard_t * shard)
+{
+ siridb_shard_decref(shard);
+}
+
+void siridb_shard_drop(siridb_shard_t * shard, siridb_t * siridb)
+{
+ siridb_series_t * series;
+ siridb_shard_t * pop_shard = NULL;
+ omap_t * shards;
+ int optimizing = 0;
+
+ uv_mutex_lock(&siridb->series_mutex);
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ shards = imap_get(siridb->shards, shard->id);
+ if (shards)
+ {
+ pop_shard = omap_rm(shards, shard->duration);
+ if (shards->n == 0)
+ {
+ free(imap_pop(siridb->shards, shard->id));
+ }
+ }
+
+ /*
+ * When optimizing, 'pop_shard' is always the new shard and 'shard'
+ * will be set to the old one.
+ */
+ if (pop_shard != NULL && (~pop_shard->flags & SIRIDB_SHARD_IS_REMOVED))
+ {
+ pop_shard->flags |= SIRIDB_SHARD_IS_REMOVED;
+ SHARD_remove(pop_shard);
+
+ if (shard != pop_shard)
+ {
+ optimizing = 1;
+ }
+ else if (shard->replacing != NULL)
+ {
+ optimizing = 1;
+ shard = shard->replacing;
+ }
+ }
+ else
+ {
+ log_warning("Shard id '%" PRIu64 "' is already dropped", shard->id);
+ }
+
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ /*
+ * We need a series mutex here since we depend on the series index
+ * and we create a copy since series might be removed when the length
+ * of series is zero after removing the shard
+ */
+
+ /*
+ * Create a copy since series might be removed and when optimizing we need
+ * to remove indexes for both the old and new shard. Since a series might
+ * be dropped by the first call to remove shard, we need an extra reference
+ * for each series.
+ */
+ if (optimizing)
+ {
+ vec_t * vec = imap_2vec_ref(siridb->series_map);
+ size_t i;
+
+ if (vec == NULL)
+ {
+ ERR_ALLOC
+ }
+ else for (i = 0; i < vec->len; i++)
+ {
+ series = (siridb_series_t *) vec->data[i];
+ if (shard->id % shard->duration == series->mask)
+ {
+ siridb_series_remove_shard(siridb, series, shard);
+ siridb_series_remove_shard(siridb, series, pop_shard);
+ }
+ siridb_series_decref(series);
+ }
+
+ vec_free(vec);
+ }
+ else
+ {
+ vec_t * vec = imap_2vec(siridb->series_map);
+ size_t i;
+
+ if (vec == NULL)
+ {
+ ERR_ALLOC
+ }
+ else for (i = 0; i < vec->len; i++)
+ {
+ series = (siridb_series_t *) vec->data[i];
+ if (shard->id % shard->duration == series->mask)
+ {
+ siridb_series_remove_shard(siridb, series, shard);
+ }
+ }
+ vec_free(vec);
+ }
+
+ if (pop_shard != NULL)
+ {
+ siridb_shard_decref(pop_shard);
+ }
+
+ uv_mutex_unlock(&siridb->series_mutex);
+}
+
+/*
+ * NEVER call this function but call siridb_shard_decref instead.
+ *
+ * Destroys a shard.
+ * When flag SIRIDB_SHARD_WILL_BE_REMOVED is set, the file will be removed.
+ *
+ * A signal can be raised in case closing the shard file fails.
+ */
+void siridb__shard_free(siridb_shard_t * shard)
+{
+ if (shard->replacing != NULL)
+ {
+ /* in case shard->replacing is set we also need to free this shard */
+ siridb_shard_decref(shard->replacing);
+ }
+
+ /* this will close the file, even when other references exist */
+ siri_fp_decref(shard->fp);
+
+ free(shard->fn);
+ free(shard);
+}
+
+/*
+ * Returns 0 when successful or a negative value in case of an error.
+ */
+static int SHARD_remove(siridb_shard_t * shard)
+{
+ int rc = 0;
+
+ if (shard->replacing != NULL)
+ {
+ rc = SHARD_remove(shard->replacing);
+ }
+ else if (shard->flags & SIRIDB_SHARD_HAS_INDEX)
+ {
+ /* We never have to delete the temporary (__) index file since this is
+ * the responsibility for the optimize task. This index file can never
+ * be in use by SiriDB. In case deletion has failed, nothing will
+ * happen because the file will not be read at startup.
+ */
+ siridb_shard_idx_file(buffer, shard->fn);
+
+ /* unlink the index file */
+ if (unlink(buffer))
+ {
+ /* not a real issue, only a warning since this is not expected */
+ log_warning("Removing index file failed: %s", buffer);
+ }
+ else
+ {
+ log_info("Index file removed: %s", buffer);
+ }
+ }
+
+ siri_fp_close(shard->fp);
+
+ rc += unlink(shard->fn);
+
+ if (rc == 0)
+ {
+ log_info("Shard file removed: %s", shard->fn);
+ }
+ else if (shard->fn != NULL)
+ {
+ log_critical(
+ "Removing shard file failed: %s (error code: %d)",
+ shard->fn,
+ rc);
+ }
+
+ return rc;
+}
+
+/*
+ * This function applies the index on the appropriate series. In case the
+ * series is not found, a log line will be displayed if this is the first
+ * one in the shard which is not found. The next series which cannot be found
+ * is simply ignored. In case the series id is not possible (invalid id),
+ * then an log error is displayed and the return value will be -1.
+ */
+static ssize_t SHARD_apply_idx(
+ siridb_t * siridb,
+ siridb_shard_t * shard,
+ char * pt,
+ size_t pos,
+ int is_ts64)
+{
+ ssize_t size;
+ uint16_t len;
+ uint32_t series_id;
+ siridb_series_t * series;
+ uint16_t cinfo = 0;
+
+ series_id = *((uint32_t *) pt);
+ if (series_id == 0)
+ {
+ return 0;
+ }
+
+ len = *((uint16_t *) (pt + (is_ts64 ? 20 : 12))); /* LEN POS IN INDEX */
+ series = imap_get(siridb->series_map, series_id);
+
+ if (shard->tp == SIRIDB_SHARD_TP_LOG)
+ {
+ cinfo = *((uint16_t *)(pt + (is_ts64 ? IDX64_SZ : IDX32_SZ)));
+ size = (ssize_t) siridb_points_get_size_log(cinfo);
+ size += len * (
+ shard->flags & SIRIDB_SHARD_IS_COMPRESSED ?
+ (len < POINTS_ZIP_THRESHOLD ?
+ sizeof(uint64_t) :
+ 0) : is_ts64 ?
+ sizeof(uint64_t) :
+ sizeof(uint32_t));
+ }
+ else if (shard->flags & SIRIDB_SHARD_IS_COMPRESSED)
+ {
+ cinfo = *((uint16_t *)(pt + (is_ts64 ? IDX64_SZ : IDX32_SZ)));
+ size = (ssize_t) siridb_points_get_size_zipped(cinfo, len);
+ }
+ else
+ {
+ size = len * (is_ts64 ? 16 : 12);
+ }
+
+ if (series == NULL)
+ {
+ if (series_id > siridb->max_series_id)
+ {
+ log_error(
+ "Unexpected Series ID %" PRIu32
+ " is found in shard %" PRIu64 " (%s) at "
+ "position %ld. This indicates that this shard is "
+ "probably corrupt. The next optimize cycle will most "
+ "likely fix this shard but you might loose some data.",
+ series_id,
+ shard->id,
+ shard->fn,
+ pos);
+ shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ return size;
+ }
+
+ /* this shard has remove series, make sure the flag is set */
+ if (!(shard->flags & SIRIDB_SHARD_HAS_DROPPED_SERIES))
+ {
+ log_debug(
+ "At least Series ID %" PRIu32
+ " is found in shard %" PRIu64 " (%s) but "
+ "does not exist anymore. We will remove the series on "
+ "the next optimize cycle.",
+ series_id,
+ shard->id,
+ shard->fn);
+ shard->flags |= SIRIDB_SHARD_HAS_DROPPED_SERIES;
+ }
+ }
+ else
+ {
+ uint64_t start_ts = is_ts64 ? /* START_TS IN HEADER */
+ (uint64_t) *((uint64_t *) (pt + 4)) :
+ (uint64_t) *((uint32_t *) (pt + 4));
+ uint64_t end_ts = is_ts64 ? /* END_TS IN HEADER */
+ (uint64_t) *((uint64_t *) (pt + 12)) :
+ (uint64_t) *((uint32_t *) (pt + 8));
+ uint64_t start = shard->id - series->mask;
+ uint64_t end = start + shard->duration;
+
+ if (start_ts < start || end_ts >= end)
+ {
+ log_error(
+ "Unexpected Time range for series ID %" PRIu32
+ " is found in shard %" PRIu64 " (%s) at "
+ "position %ld. This indicates that this shard is "
+ "probably corrupt. The next optimize cycle will most "
+ "likely fix this shard but you might loose some data.",
+ series_id,
+ shard->id,
+ shard->fn,
+ pos);
+ shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ return size;
+ }
+
+ if (siridb_series_add_idx(
+ series,
+ shard,
+ start_ts,
+ end_ts,
+ (uint32_t) pos,
+ len,
+ cinfo) == 0)
+ {
+ /* update the series length property */
+ series->length += len;
+ }
+ else
+ {
+ /* signal is raised */
+ log_critical("Cannot load index for Series ID %u", series->id);
+ }
+ }
+
+ return size;
+}
+
+/*
+ * Read an index file for a shard in case the shard has flag
+ * SIRIDB_SHARD_HAS_INDEX set. Returns 0 in case the index was read successful
+ * and if the flag was not set. Returns a negative value in case of an error.
+ *
+ * Member shard->len will be updated according the index.
+ */
+static int SHARD_get_idx(
+ siridb_t * siridb,
+ siridb_shard_t * shard,
+ int is_ts64)
+{
+ const unsigned int idx_sz = (
+ (shard->flags & SIRIDB_SHARD_IS_COMPRESSED) ||
+ (shard->tp == SIRIDB_SHARD_TP_LOG)) ?
+ (is_ts64 ? IDX64E_SZ : IDX32E_SZ) :
+ (is_ts64 ? IDX64_SZ : IDX32_SZ);
+ size_t i, n;
+ char * data, * pt;
+ ssize_t size;
+ FILE * fp;
+
+ if (~shard->flags & SIRIDB_SHARD_HAS_INDEX)
+ {
+ return 0;
+ }
+
+ n = SIRIDB_SHARD_MAX_CHUNK_SZ / idx_sz;
+ data = malloc(n * idx_sz);
+
+ if (data == NULL)
+ {
+ log_critical("Memory allocation error");
+ free(data);
+ return -1;
+ }
+
+ /* get the index file name */
+ siridb_shard_idx_file(fn, shard->fn);
+
+ fp = fopen(fn, "r");
+ if (fp == NULL)
+ {
+ log_critical("Cannot open index file for reading: '%s'", fn);
+ free(data);
+ return -1;
+ }
+
+ while ((n = fread(data, idx_sz, n, fp)))
+ {
+ pt = data;
+ for (i = 0; i < n; i++, pt += idx_sz)
+ {
+ size = SHARD_apply_idx(
+ siridb,
+ shard,
+ pt,
+ shard->len,
+ is_ts64);
+
+ if (size < 0)
+ {
+ log_critical("Error while reading index file: '%s'", fn);
+ fclose(fp);
+ free(data);
+ return -1;
+ }
+
+ shard->len += size;
+ }
+ }
+
+ fclose(fp);
+ free(data);
+
+ return 0;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ *
+ * A SIGNAL might be raised in case of memory errors. We mark the shard as
+ * corrupt in case of disk errors and try to recover on the next optimize
+ * cycle.
+ */
+static int SHARD_load_idx(
+ siridb_t * siridb,
+ siridb_shard_t * shard,
+ FILE * fp,
+ int is_ts64)
+{
+ const unsigned int idx_sz = (
+ (shard->flags & SIRIDB_SHARD_IS_COMPRESSED) ||
+ (shard->tp == SIRIDB_SHARD_TP_LOG)) ?
+ is_ts64 ? IDX64E_SZ : IDX32E_SZ :
+ is_ts64 ? IDX64_SZ : IDX32_SZ;
+
+ char idx[idx_sz];
+ ssize_t sz;
+ int rc;
+ size_t size, pos;
+
+ while ((size = fread(&idx, 1, idx_sz, fp)) == idx_sz)
+ {
+ pos = shard->len + idx_sz;
+
+ sz = SHARD_apply_idx(siridb, shard, idx, pos, is_ts64);
+ if (sz == 0)
+ {
+ break;
+ }
+ else if (sz < 0)
+ {
+ return -1;
+ }
+
+ rc = fseeko(fp, sz, SEEK_CUR); /* 16 = NUM64 point size */
+ if (rc != 0)
+ {
+ log_error(
+ "Seek error in shard %" PRIu64 " (%s) at position %ld. "
+ "Mark this shard as corrupt. The next optimize cycle will "
+ "most likely fix this shard but you might loose some data.",
+ shard->id,
+ shard->fn,
+ pos);
+ shard->flags |= SIRIDB_SHARD_IS_CORRUPT;
+ return -1;
+ }
+
+ shard->flags |= SIRIDB_SHARD_HAS_NEW_VALUES;
+ shard->len = pos + sz;
+ }
+
+ return siri_err;
+}
+
+/*
+ * Set shard->fn to the correct file name.
+ *
+ * Returns the length of 'fn' or a negative value in case of an error.
+ */
+static inline int SHARD_init_fn(siridb_t * siridb, siridb_shard_t * shard)
+{
+ return asprintf(
+ &shard->fn,
+ "%s%s%s%016"PRIX64"_%016"PRIX64".sdb",
+ siridb->dbpath,
+ SIRIDB_SHARDS_PATH,
+ (shard->replacing == NULL) ? "" : "__",
+ shard->id,
+ shard->duration);
+}
+
+/*
+ * Write a header for a chunk of points. The header can be written to argument
+ * fp which should be a pointer to the index, or the shard file.
+ *
+ * In case of an error the function returns 0, otherwise the size which is
+ * written.
+ */
+static size_t SHARD_write_header(
+ siridb_t * siridb,
+ siridb_series_t * series,
+ siridb_points_t * points,
+ uint_fast32_t start,
+ uint_fast32_t end,
+ uint16_t * cinfo,
+ FILE * fp)
+{
+ uint16_t len = end - start;
+ size_t size = sizeof(uint32_t);
+ char buf[24];
+ memcpy(buf, &series->id, sizeof(uint32_t));
+
+ switch (siridb->time->ts_sz)
+ {
+ case sizeof(uint32_t):
+ {
+ uint32_t start_ts = (uint32_t) points->data[start].ts;
+ uint32_t end_ts = (uint32_t) points->data[end - 1].ts;
+ memcpy(buf + size, &start_ts, sizeof(uint32_t));
+ size += sizeof(uint32_t);
+ memcpy(buf + size, &end_ts, sizeof(uint32_t));
+ size += sizeof(uint32_t);
+ }
+ break;
+
+ case sizeof(uint64_t):
+ memcpy(buf + size, &points->data[start].ts, sizeof(uint64_t));
+ size += sizeof(uint64_t);
+ memcpy(buf + size, &points->data[end - 1].ts, sizeof(uint64_t));
+ size += sizeof(uint64_t);
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ memcpy(buf + size, &len, sizeof(uint16_t));
+ size += sizeof(uint16_t);
+
+ if (cinfo != NULL)
+ {
+ memcpy(buf + size, cinfo, sizeof(uint16_t));
+ size += sizeof(uint16_t);
+ }
+
+ if (fwrite(buf, size, 1, fp) != 1)
+ {
+ return 0;
+ }
+
+ return size;
+}
+
+
+static int SHARD_grow(siridb_shard_t * shard)
+{
+ assert (shard->fp);
+
+ int buffer_fd = fileno(shard->fp->fp);
+
+ shard->size = ((size_t) (shard->size / SHARD_GROW_SZ) + 2) * SHARD_GROW_SZ;
+
+ if (buffer_fd == -1)
+ {
+ log_error("Cannot get file descriptor for '%s'", shard->fn);
+ return -1;
+ }
+
+ if (ftruncate(buffer_fd, (off_t) shard->size) || fsync(buffer_fd))
+ {
+ log_error("Cannot truncate shard file: '%s'", shard->fn);
+ return -1;
+ }
+
+ return 0;
+}
--- /dev/null
+/*
+ * shards.c - Collection of SiriDB shards.
+ *
+ * Info shards->mutex:
+ *
+ * Main thread:
+ * siridb->shards : read (lock) write (lock)
+
+ * Other threads:
+ * siridb->shards : read (lock) write (lock)
+ *
+ * Note: since series->idx hold a reference to a shard, a lock to the
+ * series_mutex is required in some cases.
+ */
+#include <ctype.h>
+#include <dirent.h>
+#include <logger/logger.h>
+#include <siri/db/shard.h>
+#include <siri/db/shards.h>
+#include <siri/db/series.inline.h>
+#include <siri/db/misc.h>
+#include <siri/siri.h>
+#include <stdbool.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <siri/db/db.h>
+#include <xpath/xpath.h>
+#include <omap/omap.h>
+
+#define SIRIDB_SHARD_LEN 37
+
+static bool SHARDS_must_migrate_shard(
+ char * fn,
+ const char * ext,
+ uint64_t * shard_id)
+{
+ size_t n = strlen(fn);
+ char * tmp = NULL;
+
+ if (n < 5)
+ {
+ return false;
+ }
+
+ *shard_id = strtoull(fn, &tmp, 10);
+
+ if (tmp == NULL)
+ {
+ return false;
+ }
+
+ return strcmp(tmp, ext) == 0;
+}
+
+static bool SHARDS_read_id_and_duration(
+ char * fn,
+ const char * ext,
+ uint64_t * shard_id,
+ uint64_t * duration)
+{
+ size_t n = strlen(fn);
+ char * tmp = NULL;
+
+ if (n != SIRIDB_SHARD_LEN)
+ {
+ return false;
+ }
+
+ *shard_id = strtoull(fn, &tmp, 16);
+ if (tmp == NULL)
+ {
+ return false;
+ }
+ fn = tmp;
+
+ if (*fn != '_')
+ {
+ return false;
+ }
+
+ ++fn;
+
+ *duration = strtoull(fn, &tmp, 16);
+ if (tmp == NULL)
+ {
+ return false;
+ }
+ fn = tmp;
+ return strcmp(fn, ext) == 0;
+}
+
+/*
+ * Returns true if fn is a temp shard or index filename, false if not.
+ */
+static bool SHARDS_is_temp_fn(char * fn)
+{
+ size_t n = strlen(fn);
+
+ return (n > 8 &&
+ fn[0] == '_' &&
+ fn[1] == '_' &&
+ fn[n-4] == '.' && ((
+ fn[n-3] == 's' &&
+ fn[n-2] == 'd' &&
+ fn[n-1] == 'b'
+ ) || (
+ fn[n-3] == 'i' &&
+ fn[n-2] == 'd' &&
+ fn[n-1] == 'x'
+ )));
+}
+
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * (a SIGNAL might be raised in case of an error)
+ */
+int siridb_shards_load(siridb_t * siridb)
+{
+ struct stat st;
+ struct dirent ** shard_list;
+ char buffer[XPATH_MAX];
+ int n, total, rc = 0;
+ uint64_t shard_id, duration;
+
+ memset(&st, 0, sizeof(struct stat));
+
+ log_info("Loading shards");
+
+ siridb_misc_get_fn(path, siridb->dbpath, SIRIDB_SHARDS_PATH);
+
+ if (strlen(path) >= XPATH_MAX - SIRIDB_SHARD_LEN - 1)
+ {
+ log_error("Shard path too long: '%s'", path);
+ return -1;
+ }
+
+ if (stat(path, &st) == -1)
+ {
+ log_warning(
+ "Shards directory not found, creating directory '%s'.",
+ path);
+ if (mkdir(path, 0700) == -1)
+ {
+ log_error("Cannot create directory '%s'.", path);
+ return -1;
+ }
+ }
+
+ total = scandir(path, &shard_list, NULL, alphasort);
+
+ if (total < 0)
+ {
+ /* no need to free shard_list when total < 0 */
+ log_error("Cannot read shards directory '%s'.", path);
+ return -1;
+ }
+
+ for (n = 0; n < total; n++)
+ {
+ char * base_fn = shard_list[n]->d_name;
+
+ if (SHARDS_is_temp_fn(base_fn))
+ {
+ snprintf(buffer, XPATH_MAX, "%s%s", path, base_fn);
+
+ log_warning("Removing temporary file: '%s'", buffer);
+
+ if (unlink(buffer))
+ {
+ log_error("Error while removing temporary file: '%s'", buffer);
+ rc = -1;
+ break;
+ }
+ }
+
+ if (!SHARDS_read_id_and_duration(
+ base_fn,
+ ".sdb",
+ &shard_id,
+ &duration))
+ {
+ if (SHARDS_must_migrate_shard(
+ base_fn,
+ ".sdb",
+ &shard_id))
+ {
+ log_warning("Migrate shard: '%s'", base_fn);
+ if (siridb_shard_migrate(siridb, shard_id, &duration))
+ {
+ log_error("Error while migrating shard: '%s'", base_fn);
+ rc = -1;
+ break;
+ }
+ }
+ else
+ {
+ continue;
+ }
+ }
+
+ /* we are sure this fits since the filename is checked */
+ if (siridb_shard_load(siridb, shard_id, duration))
+ {
+ log_error("Error while loading shard: '%s'", base_fn);
+ rc = -1;
+ break;
+ }
+ }
+
+ while (total--)
+ {
+ free(shard_list[total]);
+ }
+ free(shard_list);
+
+ return rc;
+}
+
+void siridb_shards_destroy_cb(omap_t * shards)
+{
+ omap_destroy(shards, (omap_destroy_cb) &siridb__shard_decref);
+}
+
+/*
+ * Returns siri_err which is 0 if successful or a negative integer in case
+ * of an error. (a SIGNAL is also raised in case of an error)
+ */
+int siridb_shards_add_points(
+ siridb_t * siridb,
+ siridb_series_t * series,
+ siridb_points_t * points)
+{
+ _Bool is_num = siridb_series_isnum(series);
+ siridb_shard_t * shard;
+ omap_t * shards;
+ uint64_t duration = siridb_series_duration(series);
+
+ uv_mutex_lock(&siridb->values_mutex);
+
+ uint64_t expire_at = is_num ? siridb->exp_at_num : siridb->exp_at_log;
+
+ if (duration == 0)
+ {
+ uint64_t interval = siri.cfg->shard_auto_duration
+ ? siridb_points_get_interval(points)
+ : 0;
+
+ duration = interval
+ ? siridb_shard_duration_from_interval(siridb, interval)
+ : is_num
+ ? siridb->duration_num
+ : siridb->duration_log;
+ }
+
+ uv_mutex_unlock(&siridb->values_mutex);
+
+ uint64_t shard_start, shard_end, shard_id;
+ uint_fast32_t start, end, num_chunks, pstart, pend;
+ uint16_t chunk_sz;
+ uint16_t cinfo = 0;
+ size_t size, pos;
+
+ for (end = 0; end < points->len;)
+ {
+ shard_start = points->data[end].ts / duration * duration;
+ shard_end = shard_start + duration;
+ shard_id = shard_start + series->mask;
+
+ for ( start = end;
+ end < points->len && points->data[end].ts < shard_end;
+ end++);
+
+ if (shard_end < expire_at)
+ {
+ series->length -= end - start;
+ series_update_start_end(series);
+ continue;
+ }
+
+ shard = NULL;
+ shards = imap_get(siridb->shards, shard_id);
+ if (shards != NULL)
+ {
+ shard = omap_get(shards, duration);
+ /* shard may be NULL if no shard according the duration is found */
+ }
+ else
+ {
+ shards = omap_create();
+ if (shards == NULL || imap_add(siridb->shards, shard_id, shards))
+ {
+ ERR_ALLOC
+ return -1; /* might leak a few bytes */
+ }
+ }
+
+ if (shard == NULL)
+ {
+ shard = siridb_shard_create(
+ siridb,
+ shards,
+ shard_id,
+ duration,
+ is_num ? SIRIDB_SHARD_TP_NUMBER : SIRIDB_SHARD_TP_LOG,
+ NULL);
+ if (shard == NULL)
+ {
+ return -1; /* signal is raised */
+ }
+ }
+
+ if (start != end)
+ {
+ size = end - start;
+
+ num_chunks = (size - 1) / shard->max_chunk_sz + 1;
+ chunk_sz = size / num_chunks + (size % num_chunks != 0);
+
+ for (pstart = start; pstart < end; pstart += chunk_sz)
+ {
+ pend = pstart + chunk_sz;
+ if (pend > end)
+ {
+ pend = end;
+ }
+
+ if ((pos = siridb_shard_write_points(
+ siridb,
+ series,
+ shard,
+ points,
+ pstart,
+ pend,
+ NULL,
+ &cinfo)) == 0)
+ {
+ log_critical(
+ "Could not write points to shard '%s'",
+ shard->fn);
+ }
+ else
+ {
+ siridb_series_add_idx(
+ series,
+ shard,
+ points->data[pstart].ts,
+ points->data[pend - 1].ts,
+ pos,
+ pend - pstart,
+ cinfo);
+ if (shard->replacing != NULL)
+ {
+ siridb_shard_write_points(
+ siridb,
+ series,
+ shard->replacing,
+ points,
+ pstart,
+ pend,
+ NULL,
+ &cinfo);
+ }
+ }
+ }
+ }
+ }
+ return siri_err;
+}
+
+static inline int SHARDS_count_cb(omap_t * omap, size_t * n)
+{
+ *n += omap->n;
+ return 0;
+}
+
+size_t siridb_shards_n(siridb_t * siridb)
+{
+ size_t n = 0;
+ imap_walk(siridb->shards, (imap_cb) SHARDS_count_cb, &n);
+ return n;
+}
+
+static inline int SHARDS_to_vec_cb(omap_t * omap, vec_t * v)
+{
+ omap_iter_t iter = omap_iter(omap);
+ siridb_shard_t * shard;
+ for (;iter && (shard = iter->data_); iter = iter->next_)
+ {
+ vec_append(v, shard);
+ siridb_shard_incref(shard);
+ }
+ return 0;
+}
+
+vec_t * siridb_shards_vec(siridb_t * siridb)
+{
+ size_t n = siridb_shards_n(siridb);
+ vec_t * vec = vec_new(n);
+ if (vec == NULL)
+ {
+ return NULL;
+ }
+ (void) imap_walk(siridb->shards, (imap_cb) SHARDS_to_vec_cb, vec);
+ return vec;
+}
+
+double siridb_shards_count_percent(
+ siridb_t * siridb,
+ uint64_t end_ts,
+ uint8_t tp)
+{
+ size_t i;
+ double percent = 1.0;
+ vec_t * shards_list = NULL;
+ size_t count = 0;
+ size_t total = 0;
+ uint64_t duration = tp == SIRIDB_SHARD_TP_NUMBER
+ ? siridb->duration_num
+ : siridb->duration_log;
+
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ shards_list = siridb_shards_vec(siridb);
+
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ if (shards_list == NULL || shards_list->len == 0)
+ {
+ free(shards_list);
+ return 0.0;
+ }
+
+ for (i = 0; i < shards_list->len; i++)
+ {
+ siridb_shard_t * shard = (siridb_shard_t *) shards_list->data[i];
+ if (shard->tp == tp)
+ {
+ ++total;
+ count += ((shard->id - shard->id % duration) + duration) < end_ts;
+ }
+ siridb_shard_decref(shard);
+ }
+
+ percent = total ? (double) count / (double) total : 0.0;
+ free(shards_list);
+ return percent;
+}
--- /dev/null
+/*
+ * sset.c - Set operations on series.
+ */
+
+#include <assert.h>
+#include <siri/db/sset.h>
+#include <stdlib.h>
+#include <siri/db/series.h>
+
+siridb_sset_t * siridb_sset_new(imap_t * series_map, imap_update_cb update_cb)
+{
+ siridb_sset_t * sset = malloc(sizeof(siridb_sset_t));
+ if (sset == NULL)
+ {
+ return NULL;
+ }
+
+ sset->series_map = series_map;
+ sset->update_cb = update_cb;
+
+ return sset;
+}
+
+void siridb_sset_free(siridb_sset_t * sset)
+{
+ if (sset == NULL)
+ {
+ return;
+ }
+
+ if (sset->series_map)
+ {
+ imap_free(sset->series_map, (imap_free_cb) &siridb__series_decref);
+ }
+
+ free(sset);
+}
--- /dev/null
+/*
+ * tag.c - Tag.
+ *
+ * author : Jeroen van der Heijden
+ * email : jeroen@transceptor.technology
+ * copyright : 2017, Transceptor Technology
+ *
+ * changes
+ * - initial version, 16-06-2017
+ *
+ */
+#define _GNU_SOURCE
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/tag.h>
+#include <stdlib.h>
+#include <siri/db/series.h>
+#include <ctype.h>
+#include <uv.h>
+#include <unistd.h>
+#include <siri/grammar/grammar.h>
+
+#define TAGFN_NUMBERS 9
+#define SIRIDB_MIN_TAG_LEN 1
+#define SIRIDB_MAX_TAG_LEN 255
+
+/*
+ * Returns tag when successful or NULL in case of an error.
+ */
+siridb_tag_t * siridb_tag_new(siridb_tags_t * tags, uint64_t id)
+{
+ siridb_tag_t * tag = (siridb_tag_t *) malloc(sizeof(siridb_tag_t));
+ if (tag != NULL)
+ {
+ tag->ref = 1;
+ tag->flags = 0;
+ tag->id = id;
+ tag->name = NULL;
+ tag->tags = tags;
+ tag->series = imap_new();
+ }
+ return tag;
+}
+
+char * siridb_tag_fn(siridb_tag_t * tag)
+{
+ char * fn;
+ if (asprintf(&fn, "%s%09"PRIx64".tag", tag->tags->path, tag->id) < 0)
+ {
+ return NULL;
+ }
+ return fn;
+}
+
+int siridb_tag_check_name(const char * name, char * err_msg)
+{
+ if (strlen(name) < SIRIDB_MIN_TAG_LEN)
+ {
+ sprintf(err_msg, "Tag name should be at least %d characters.",
+ SIRIDB_MIN_TAG_LEN);
+ return 1;
+ }
+
+ if (strlen(name) > SIRIDB_MAX_TAG_LEN)
+ {
+ sprintf(err_msg, "Tag name should be at most %d characters.",
+ SIRIDB_MAX_TAG_LEN);
+ return 1;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns tag when successful or NULL in case of an error.
+ */
+siridb_tag_t * siridb_tag_load(siridb_t * siridb, const char * _fn)
+{
+ char * fn;
+ qp_obj_t qp_tn;
+ qp_obj_t qp_series_id;
+ uint64_t series_id;
+ siridb_series_t * series;
+ qp_unpacker_t * unpacker;
+ siridb_tag_t * tag = siridb_tag_new(siridb->tags, (uint32_t) atoll(_fn));
+
+ if (tag == NULL)
+ {
+ log_critical("Memory allocation error");
+ return NULL;
+ }
+
+ fn = siridb_tag_fn(tag);
+ if (fn == NULL)
+ {
+ log_critical("Memory allocation error");
+ goto fail0;
+ }
+
+ unpacker = qp_unpacker_ff(fn);
+ if (unpacker == NULL)
+ {
+ log_critical("Cannot open tag file for reading: %s", fn);
+ goto fail1;
+ }
+
+
+ if (!qp_is_array(qp_next(unpacker, NULL)) ||
+ qp_next(unpacker, &qp_tn) != QP_RAW ||
+ (tag->name = strndup((const char *) qp_tn.via.raw, qp_tn.len)) == NULL)
+ {
+ /* or a memory allocation error, but the same result */
+ log_critical("Expected an array with a tag name in file: %s", fn);
+ goto fail2;
+ }
+
+
+ while (qp_next(unpacker, &qp_series_id) == QP_INT64)
+ {
+ series_id = (uint64_t) qp_series_id.via.int64;
+ series = imap_get(siridb->series_map, series_id);
+
+ if (series == NULL)
+ {
+ siridb_tags_set_require_save(siridb->tags, tag);
+
+ log_error(
+ "Cannot find series id %" PRId64
+ " which was tagged with '%s'",
+ qp_series_id.via.int64,
+ tag->name);
+ }
+ else if (imap_add(tag->series, series_id, series) == 0)
+ {
+ siridb_series_incref(series);
+ }
+ else
+ {
+ log_error(
+ "Cannot add series '%s' to tag '%s'",
+ series->name,
+ tag->name);
+ }
+ }
+
+ qp_unpacker_ff_free(unpacker);
+ free(fn);
+ return tag;
+
+fail2:
+ qp_unpacker_ff_free(unpacker);
+fail1:
+ free(fn);
+fail0:
+ siridb__tag_free(tag);
+ return NULL;
+}
+
+
+
+static int tag__save_cb(siridb_series_t * series, qp_fpacker_t * fpacker)
+{
+ return qp_fadd_int64(fpacker, (int64_t) series->id);
+}
+
+/*
+ * Lock is required
+ */
+int siridb_tag_save(siridb_tag_t * tag)
+{
+ int rc = -1;
+ qp_fpacker_t * fpacker;
+
+ char * fn = siridb_tag_fn(tag);
+ if (fn == NULL)
+ {
+ return rc;
+ }
+
+ fpacker = qp_open(fn, "w");
+ if (fpacker == NULL)
+ {
+ goto fail0;
+ }
+
+ if (/* open a new array */
+ qp_fadd_type(fpacker, QP_ARRAY_OPEN) ||
+
+ /* write the tag name */
+ qp_fadd_string(fpacker, tag->name))
+ {
+ goto fail1;
+ }
+
+ rc = imap_walk(tag->series, (imap_cb) tag__save_cb, fpacker);
+
+fail1:
+ rc = qp_close(fpacker) || rc;
+
+fail0:
+ free(fn);
+ return rc;
+}
+
+/*
+ * Returns true when the given property (CLERI keyword) needs a remote query
+ */
+int siridb_tag_is_remote_prop(uint32_t prop)
+{
+ return (prop == CLERI_GID_K_SERIES) ? 1 : 0;
+}
+
+/*
+ * This function can raise a SIGNAL. In this case the packer is not filled
+ * with the correct values.
+ */
+void siridb_tag_prop(siridb_tag_t * tag, qp_packer_t * packer, int prop)
+{
+ switch (prop)
+ {
+ case CLERI_GID_K_NAME:
+ qp_add_string(packer, tag->name);
+ break;
+ case CLERI_GID_K_SERIES:
+ qp_add_int64(packer, (int64_t) tag->n);
+ break;
+ }
+}
+
+int siridb_tag_cexpr_cb(siridb_tag_t * tag, cexpr_condition_t * cond)
+{
+ switch (cond->prop)
+ {
+ case CLERI_GID_K_NAME:
+ return cexpr_str_cmp(cond->operator, tag->name, cond->str);
+ case CLERI_GID_K_SERIES:
+ return cexpr_int_cmp(cond->operator, (int64_t) tag->n, cond->int64);
+ }
+
+ log_critical("Unknown group property received: %d", cond->prop);
+ assert (0);
+ return -1;
+}
+
+int siridb_tag_set_name(
+ siridb_t * siridb,
+ siridb_tag_t * tag,
+ const char * name,
+ char * err_msg)
+{
+ if (siridb_tag_check_name(name, err_msg) != 0)
+ {
+ return 1;
+ }
+
+ if (ct_get(siridb->tags->tags, name) != NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Tag '%s' already exists.",
+ name);
+ return 1;
+ }
+
+ if (siridb->groups && ct_get(siridb->groups->groups, name) != NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Group '%s' already exists.",
+ name);
+ return 1;
+ }
+
+ if (tag->name != NULL)
+ {
+ int rc;
+
+ /* group already exists */
+ uv_mutex_lock(&siridb->tags->mutex);
+
+ rc = ( ct_pop(siridb->tags->tags, tag->name) == NULL ||
+ ct_add(siridb->tags->tags, name, tag));
+
+ uv_mutex_unlock(&siridb->tags->mutex);
+
+ if (rc)
+ {
+ ERR_C
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Critical error while replacing tag name '%s' with '%s' "
+ "in tree.",
+ tag->name,
+ name);
+ return -1;
+ }
+ }
+
+ free(tag->name);
+ tag->name = strdup(name);
+
+ if (tag->name == NULL)
+ {
+ ERR_ALLOC
+ sprintf(err_msg, "Memory allocation error.");
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Can be used as a callback, in other cases go for the macro.
+ */
+void siridb__tag_decref(siridb_tag_t * tag)
+{
+ siridb_tag_decref(tag);
+}
+
+/*
+ * NEVER call this function but rather call siridb_tag_decref instead.
+ *
+ * Destroy a tag object. Parsing NULL is not allowed.
+ */
+void siridb__tag_free(siridb_tag_t * tag)
+{
+#ifdef DEBUG
+ log_debug("Free tag: '%s'", tag->name);
+#endif
+
+ if ((tag->flags & TAG_FLAG_CLEANUP))
+ {
+ char * fn = siridb_tag_fn(tag);
+ if (!fn || unlink(fn))
+ {
+ log_critical("Cannot remove tag (file): '%s'", tag->name);
+ }
+ free(fn);
+ }
+ else if ((tag->flags & TAG_FLAG_REQUIRE_SAVE) && siridb_tag_save(tag))
+ {
+ log_critical("Cannot save tag '%s'", tag->name);
+ }
+
+ free(tag->name);
+
+ if (tag->series != NULL)
+ {
+ imap_free(tag->series, (imap_free_cb) siridb__series_decref);
+ }
+
+ free(tag);
+}
+
+/*
+ * Returns 1 (true) if the file name is valid and 0 (false) if not
+ */
+int siridb_tag_is_valid_fn(const char * fn)
+{
+ int i = 0;
+ while (*fn && isdigit(*fn))
+ {
+ fn++;
+ i++;
+ }
+ return (i == TAGFN_NUMBERS) ? (strcmp(fn, ".tag") == 0) : 0;
+}
--- /dev/null
+/*
+ * tags.c - Tags.
+ *
+ * author : Jeroen van der Heijden
+ * email : jeroen@transceptor.technology
+ * copyright : 2017, Transceptor Technology
+ *
+ * changes
+ * - initial version, 16-06-2017
+ *
+ */
+#define _GNU_SOURCE
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/tags.h>
+#include <stdlib.h>
+#include <vec/vec.h>
+#include <siri/db/series.h>
+#include <siri/net/protocol.h>
+#include <unistd.h>
+#include <siri/siri.h>
+
+static int TAGS_load(siridb_t * siridb);
+static int TAGS_dropped_series(
+ siridb_tag_t * tag,
+ void * data __attribute__((unused)));
+static int TAGS_nseries(
+ siridb_tag_t * tag,
+ void * data __attribute__((unused)));
+
+/*
+ * Initialize tags. Returns 0 if successful or -1 in case of an error.
+ */
+int siridb_tags_init(siridb_t * siridb)
+{
+ log_info("Loading tags");
+ siridb->tags = (siridb_tags_t *) malloc(sizeof(siridb_tags_t));
+ if (siridb->tags == NULL)
+ {
+ return -1;
+ }
+ siridb->tags->flags = 0;
+ siridb->tags->ref = 1;
+ siridb->tags->next_id = 0;
+ siridb->tags->tags = ct_new();
+
+ uv_mutex_init(&siridb->tags->mutex);
+
+ if (asprintf(
+ &siridb->tags->path,
+ "%s%s",
+ siridb->dbpath,
+ SIRIDB_TAGS_PATH) < 0 ||
+ siridb->tags->tags == NULL ||
+ TAGS_load(siridb))
+ {
+ siridb__tags_free(siridb->tags);
+ siridb->tags = NULL;
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Main thread.
+ *
+ * Returns 0 if successful or -1 when the group is not found.
+ * (in case not found an error message is set)
+ *
+ * Note: when saving the groups to disk has failed, we log critical but
+ * the function still returns 0;
+ */
+int siridb_tags_drop_tag(
+ siridb_tags_t * tags,
+ const char * name,
+ char * err_msg)
+{
+ uv_mutex_lock(&tags->mutex);
+
+ siridb_tag_t * tag = (siridb_tag_t *) ct_pop(tags->tags, name);
+
+ uv_mutex_unlock(&tags->mutex);
+
+ if (tag == NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Tag '%s' does not exist.",
+ name);
+ return -1;
+ }
+
+ tag->flags |= TAG_FLAG_CLEANUP;
+ siridb_tag_decref(tag);
+
+ return 0;
+}
+
+siridb_tag_t * siridb_tags_add_n(
+ siridb_tags_t * tags,
+ const char * name,
+ size_t name_len)
+{
+ siridb_tag_t * tag = siridb_tag_new(tags, tags->next_id++);
+
+ if (tag != NULL)
+ {
+ tag->name = strndup(name, name_len);
+ if (tag->name == NULL || ct_add(tags->tags, tag->name, tag))
+ {
+ siridb_tag_decref(tag);
+ tag = NULL;
+ }
+ }
+ return tag;
+}
+
+siridb_tag_t * siridb_tags_add(siridb_tags_t * tags, const char * name)
+{
+ siridb_tag_t * tag = siridb_tag_new(tags, tags->next_id++);
+
+ if (tag != NULL)
+ {
+ tag->name = strdup(name);
+ if (tag->name == NULL || ct_add(tags->tags, name, tag))
+ {
+ siridb_tag_decref(tag);
+ tag = NULL;
+ }
+ }
+ return tag;
+}
+
+
+/*
+ * This function is called from the "Group" thread.
+ */
+void siridb_tags_dropped_series(siridb_tags_t * tags)
+{
+ uv_mutex_lock(&tags->mutex);
+
+ ct_values(tags->tags, (ct_val_cb) TAGS_dropped_series, tags);
+
+ tags->flags &= ~TAGS_FLAG_DROPPED_SERIES;
+
+ uv_mutex_unlock(&tags->mutex);
+}
+
+static int TAGS__save_cb(
+ siridb_tag_t * tag,
+ void * data __attribute__((unused)))
+{
+ if (siridb_tag_save(tag) == 0)
+ {
+ tag->flags &= ~ TAG_FLAG_REQUIRE_SAVE;
+ usleep(10000); // 10ms
+ return 0;
+ }
+ return 1;
+}
+
+void siridb_tags_save(siridb_tags_t * tags)
+{
+ uv_mutex_lock(&tags->mutex);
+
+ if (ct_values(tags->tags, (ct_val_cb) TAGS__save_cb, NULL) == 0)
+ {
+ tags->flags &= ~TAGS_FLAG_REQUIRE_SAVE;
+ }
+
+ uv_mutex_unlock(&tags->mutex);
+}
+
+/*
+ * Initialize each 'n' group property with the local value.
+ */
+void siridb_tags_init_nseries(siridb_tags_t * tags)
+{
+ ct_values(tags->tags, (ct_val_cb) TAGS_nseries, NULL);
+}
+
+/*
+ * Main thread.
+ */
+static int TAGS_pkg(siridb_tag_t * tag, qp_packer_t * packer)
+{
+ int rc = 0;
+ rc += qp_add_type(packer, QP_ARRAY2);
+ rc += qp_add_string_term(packer, tag->name);
+ rc += qp_add_int64(packer, tag->series->len);
+ return rc;
+}
+
+/*
+ * Main thread.
+ *
+ * Returns NULL and raises a signal in case of an error.
+ */
+sirinet_pkg_t * siridb_tags_pkg(siridb_tags_t * tags, uint16_t pid)
+{
+ qp_packer_t * packer = sirinet_packer_new(8192);
+ int rc;
+
+ if (packer == NULL || qp_add_type(packer, QP_ARRAY_OPEN))
+ {
+ return NULL; /* signal is raised */
+ }
+
+ rc = ct_values(tags->tags, (ct_val_cb) TAGS_pkg, packer);
+
+ if (rc)
+ {
+ /* signal is raised when not 0 */
+ qp_packer_free(packer);
+ return NULL;
+ }
+
+ return sirinet_packer2pkg(packer, pid, BPROTO_RES_TAGS);
+}
+
+typedef struct
+{
+ qp_packer_t * packer;
+ uint64_t id;
+} TAGS_series_t;
+
+/*
+ * Main thread.
+ */
+static int TAGS_series_pkg(siridb_tag_t * tag, TAGS_series_t * w)
+{
+ return imap_get(tag->series, w->id)
+ ? qp_add_string(w->packer, tag->name) == 0
+ : 0;
+}
+
+/*
+ * Main thread.
+ *
+ * Returns NULL in case of an error or no tags.
+ */
+sirinet_pkg_t * siridb_tags_series(siridb_series_t * series)
+{
+ TAGS_series_t w = {
+ .packer = sirinet_packer_new(1024),
+ .id = series->id,
+ };
+
+ if (w.packer == NULL ||
+ qp_add_type(w.packer, QP_ARRAY_OPEN) ||
+ qp_add_string_term_n(w.packer, series->name, series->name_len))
+ {
+ return NULL;
+ }
+
+ if (ct_values(
+ series->siridb->tags->tags,
+ (ct_val_cb) TAGS_series_pkg,
+ &w) == 0)
+ {
+ qp_packer_free(w.packer);
+ return NULL;
+ }
+
+ return sirinet_packer2pkg(w.packer, 0, BPROTO_SERIES_TAGS);
+}
+
+/*
+ * Main thread.
+ */
+static int TAGS_empty_tag_pkg(siridb_tag_t * tag, qp_packer_t * packer)
+{
+ return tag->series->len == 0
+ ? qp_add_string(packer, tag->name) == 0
+ : 0;
+}
+
+/*
+ * Main thread.
+ *
+ * Returns NULL in case of an error or no empty tags.
+ */
+sirinet_pkg_t * siridb_tags_empty(siridb_tags_t * tags)
+{
+ qp_packer_t * packer = sirinet_packer_new(1024);
+
+ if (packer == NULL || qp_add_type(packer, QP_ARRAY_OPEN))
+ {
+ return NULL;
+ }
+
+ if (ct_values(
+ tags->tags,
+ (ct_val_cb) TAGS_empty_tag_pkg,
+ packer) == 0)
+ {
+ qp_packer_free(packer);
+ return NULL;
+ }
+
+ return sirinet_packer2pkg(packer, 0, BPROTO_EMPTY_TAGS);
+}
+
+/*
+ * Main thread.
+ */
+static int TAGS_nseries(
+ siridb_tag_t * tag,
+ void * data __attribute__((unused)))
+{
+ tag->n = tag->series->len;
+ return 0;
+}
+
+/*
+ * This function is called from the "Group" thread.
+ */
+static int TAGS_dropped_series(
+ siridb_tag_t * tag,
+ void * data __attribute__((unused)))
+{
+ vec_t * tag_series = imap_vec_pop(tag->series);
+ siridb_series_t * series;
+
+ if (tag_series != NULL)
+ {
+ size_t i;
+ for (i = 0; i < tag_series->len; i++)
+ {
+ series = (siridb_series_t *) tag_series->data[i];
+ if ((series->flags & SIRIDB_SERIES_IS_DROPPED) &&
+ imap_pop(tag->series, series->id))
+ {
+ siridb_series_decref(series);
+ siridb_tags_set_require_save(tag->tags, tag);
+ }
+ }
+
+ vec_free(tag_series);
+ }
+
+ usleep(10000); // 10ms
+
+ return 0;
+}
+
+static int TAGS_load(siridb_t * siridb)
+{
+ struct stat st = {0};
+ struct dirent ** tags_list;
+ int total, n, rc;
+ siridb_tag_t * tag;
+
+ if (stat(siridb->tags->path, &st) == -1)
+ {
+ log_warning(
+ "Tags directory not found, creating directory '%s'.",
+ siridb->tags->path);
+ if (mkdir(siridb->tags->path, 0700) == -1)
+ {
+ log_error("Cannot create directory '%s'.", siridb->tags->path);
+ return -1;
+ }
+ }
+
+ total = scandir(siridb->tags->path, &tags_list, NULL, alphasort);
+
+ if (total < 0)
+ {
+ /* no need to free tags_list when total < 0 */
+ log_error("Cannot read tags directory '%s'.", siridb->tags->path);
+ return -1;
+ }
+
+ rc = 0;
+
+ for (n = 0; n < total; n++)
+ {
+ if (!siridb_tag_is_valid_fn(tags_list[n]->d_name))
+ {
+ continue;
+ }
+
+ /* we are sure this fits since the filename is checked */
+ tag = siridb_tag_load(siridb, tags_list[n]->d_name);
+ if (tag == NULL)
+ {
+ log_error("Error while loading tag: '%s'", tags_list[n]->d_name);
+ rc = -1;
+ break;
+ }
+
+ if (ct_add(siridb->tags->tags, tag->name, tag))
+ {
+ log_error("Cannot add tag to collection");
+ siridb_tag_decref(tag);
+ rc = -1;
+ break;
+ }
+
+ if (tag->id >= siridb->tags->next_id)
+ {
+ siridb->tags->next_id = tag->id + 1;
+ }
+ }
+
+ while (total--)
+ {
+ free(tags_list[total]);
+ }
+
+ free(tags_list);
+
+ return rc;
+}
+
+void siridb__tags_free(siridb_tags_t * tags)
+{
+ if (tags->flags & TAGS_FLAG_REQUIRE_SAVE)
+ {
+ siridb_tags_save(tags);
+ }
+
+ uv_mutex_lock(&tags->mutex);
+
+ if (tags->tags != NULL)
+ {
+ ct_free(tags->tags, (ct_free_cb) siridb__tag_decref);
+ }
+
+ uv_mutex_unlock(&tags->mutex);
+ uv_mutex_destroy(&tags->mutex);
+
+ free(tags->path);
+ free(tags);
+}
--- /dev/null
+/*
+ * tasks.c - Counters for info on SiriDB tasks.
+ */
+#include <siri/db/tasks.h>
+#include <timeit/timeit.h>
+
+void siridb_tasks_init(siridb_tasks_t * tasks)
+{
+ tasks->active = 0;
+ tasks->idle_time = 0.0f;
+ timeit_start(&tasks->_timeit);
+}
--- /dev/null
+/*
+ * tee.c - To tee the data for a SiriDB database.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <siri/db/tee.h>
+#include <siri/siri.h>
+#include <siri/net/pipe.h>
+#include <logger/logger.h>
+
+#define TEE__BUF_SZ 512
+static char tee__buf[TEE__BUF_SZ];
+
+static void tee__runtime_init(uv_pipe_t * pipe);
+static void tee__write_cb(uv_write_t * req, int status);
+static void tee__on_connect(uv_connect_t * req, int status);
+static void tee__close_cb(uv_pipe_t * pipe);
+static void tee__alloc_buffer(
+ uv_handle_t * handle,
+ size_t suggsz,
+ uv_buf_t * buf);
+static void tee__on_data(
+ uv_stream_t * client,
+ ssize_t nread,
+ const uv_buf_t * buf);
+
+siridb_tee_t * siridb_tee_new(void)
+{
+ siridb_tee_t * tee = malloc(sizeof(siridb_tee_t));
+ if (tee == NULL)
+ {
+ return NULL;
+ }
+ tee->pipe_name_ = NULL;
+ tee->err_msg_ = NULL;
+ tee->pipe.data = tee;
+ tee->flags = SIRIDB_TEE_FLAG;
+ return tee;
+}
+
+void siridb_tee_free(siridb_tee_t * tee)
+{
+ free(tee->err_msg_);
+ free(tee->pipe_name_);
+ free(tee);
+}
+
+int siridb_tee_connect(siridb_tee_t * tee)
+{
+ if (tee->flags & SIRIDB_TEE_FLAG_CONNECTING)
+ {
+ return 0;
+ }
+ tee->flags |= SIRIDB_TEE_FLAG_CONNECTING;
+ uv_connect_t * req = malloc(sizeof(uv_connect_t));
+ if (req == NULL)
+ {
+ return -1;
+ }
+
+ req->data = tee;
+
+ if (uv_pipe_init(siri.loop, &tee->pipe, 0))
+ {
+ tee->flags &= ~SIRIDB_TEE_FLAG_CONNECTING;
+ free(req);
+ return -1;
+ }
+ tee->flags |= SIRIDB_TEE_FLAG_INIT;
+ tee->pipe.data = tee;
+
+ uv_pipe_connect(req, &tee->pipe, tee->pipe_name_, tee__on_connect);
+ return 0;
+}
+
+int siridb_tee_set_pipe_name(siridb_tee_t * tee, const char * pipe_name)
+{
+ free(tee->pipe_name_);
+ free(tee->err_msg_);
+ tee->err_msg_ = NULL;
+
+ if (pipe_name == NULL)
+ {
+ tee->pipe_name_ = NULL;
+
+ if (tee->flags & SIRIDB_TEE_FLAG_CONNECTED)
+ {
+ uv_close((uv_handle_t *) &tee->pipe, (uv_close_cb) tee__close_cb);
+ }
+ return 0;
+ }
+
+ tee->pipe_name_ = strdup(pipe_name);
+ if (!tee->pipe_name_)
+ {
+ return -1;
+ }
+ if (tee->flags & SIRIDB_TEE_FLAG_INIT)
+ {
+ uv_close((uv_handle_t *) &tee->pipe, (uv_close_cb) tee__runtime_init);
+ }
+ else
+ {
+ tee__runtime_init(&tee->pipe);
+ }
+ return 0;
+}
+
+void siridb_tee_write(siridb_tee_t * tee, sirinet_promise_t * promise)
+{
+ uv_write_t * req = malloc(sizeof(uv_write_t));
+ if (!req)
+ {
+ log_error("Cannot allocate memory for tee request");
+ return;
+ }
+
+ req->data = promise;
+ sirinet_promise_incref(promise);
+
+ uv_buf_t wrbuf = uv_buf_init(
+ (char *) promise->pkg,
+ sizeof(sirinet_pkg_t) + promise->pkg->len);
+
+ if (uv_write(req, (uv_stream_t *) &tee->pipe, &wrbuf, 1, tee__write_cb))
+ {
+ log_error("Cannot write to tee");
+ sirinet_promise_decref(promise);
+ }
+}
+
+const char * tee_str(siridb_tee_t * tee)
+{
+ if (tee->err_msg_)
+ {
+ return tee->err_msg_;
+ }
+ if (tee->pipe_name_)
+ {
+ return tee->pipe_name_;
+ }
+ return "disabled";
+}
+
+
+static void tee__runtime_init(uv_pipe_t * pipe)
+{
+ siridb_tee_t * tee = pipe->data;
+
+ tee->flags &= ~SIRIDB_TEE_FLAG_INIT;
+ tee->flags &= ~SIRIDB_TEE_FLAG_CONNECTING;
+ tee->flags &= ~SIRIDB_TEE_FLAG_CONNECTED;
+
+ if (siridb_tee_connect(tee))
+ {
+ log_error("Could not connect to tee at runtime");
+ }
+}
+
+static void tee__close_cb(uv_pipe_t * pipe)
+{
+ siridb_tee_t * tee = pipe->data;
+
+ tee->flags &= ~SIRIDB_TEE_FLAG_INIT;
+ tee->flags &= ~SIRIDB_TEE_FLAG_CONNECTING;
+ tee->flags &= ~SIRIDB_TEE_FLAG_CONNECTED;
+}
+
+static void tee__write_cb(uv_write_t * req, int status)
+{
+ sirinet_promise_t * promise = req->data;
+ sirinet_promise_decref(promise);
+ if (status)
+ {
+ log_error("Socket (tee) write error: %s", uv_strerror(status));
+ }
+ free(req);
+}
+
+static void tee__on_connect(uv_connect_t * req, int status)
+{
+ siridb_tee_t * tee = req->data;
+
+ if (status == 0)
+ {
+ log_info("Connection created to pipe: '%s'", tee->pipe_name_);
+ if (uv_read_start(req->handle, tee__alloc_buffer, tee__on_data))
+ {
+ free(tee->err_msg_);
+ if (asprintf(&tee->err_msg_,
+ "Cannot open pipe '%s' for reading",
+ tee->pipe_name_) >= 0)
+ {
+ log_warning(tee->err_msg_);
+ }
+ goto fail;
+ }
+ tee->flags |= SIRIDB_TEE_FLAG_CONNECTED;
+ goto done;
+ }
+
+ free(tee->err_msg_);
+ tee->err_msg_ = NULL;
+
+ if (asprintf(
+ &tee->err_msg_,
+ "Cannot connect to pipe '%s' (%s)",
+ tee->pipe_name_,
+ uv_strerror(status)) >= 0)
+ {
+ log_warning(tee->err_msg_);
+ }
+
+fail:
+ uv_close((uv_handle_t *) req->handle, (uv_close_cb) tee__close_cb);
+done:
+ free(req);
+}
+
+static void tee__alloc_buffer(
+ uv_handle_t * handle __attribute__((unused)),
+ size_t suggsz __attribute__((unused)),
+ uv_buf_t * buf)
+{
+ buf->base = tee__buf;
+ buf->len = TEE__BUF_SZ;
+}
+
+
+
+static void tee__on_data(
+ uv_stream_t * client,
+ ssize_t nread,
+ const uv_buf_t * buf __attribute__((unused)))
+{
+ if (nread < 0)
+ {
+ if (nread != UV_EOF)
+ {
+ log_error("Read error on pipe '%s' : '%s'",
+ sirinet_pipe_name((uv_pipe_t * ) client),
+ uv_err_name(nread));
+ }
+ log_info("Disconnected from tee");
+ uv_close((uv_handle_t *) client, (uv_close_cb) tee__close_cb);
+ }
+
+ if (nread > 0)
+ {
+ log_debug("Got %zd bytes on tee which will be ignored", nread);
+ }
+}
--- /dev/null
+/*
+ * time.c - Time- and time precision functions and constants.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/time.h>
+#include <siri/err.h>
+#include <stddef.h>
+#include <xmath/xmath.h>
+
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ *
+ * Node: can be destroyed be using free().
+ */
+siridb_time_t * siridb_time_new(siridb_timep_t precision)
+{
+ siridb_time_t * time = malloc(sizeof(siridb_time_t));
+ if (time == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ time->precision = precision;
+ time->factor = xmath_ipow(1000, precision);
+ time->ts_sz = (precision == SIRIDB_TIME_SECONDS) ?
+ sizeof(uint32_t) : sizeof(uint64_t);
+ }
+ return time;
+}
+
+uint64_t siridb_time_parse(const char * str, size_t len)
+{
+ uint64_t ts = atoll(str);
+ switch (str[len - 1])
+ {
+ case 's':
+ return ts;
+ case 'm':
+ return ts * 60;
+ case 'h':
+ return ts * 3600;
+ case 'd':
+ return ts * 86400;
+ case 'w':
+ return ts * 604800;
+ }
+ /* we should NEVER get here */
+ log_critical("Unexpected time char received: '%c'", str[len - 1]);
+ assert (0);
+ return 0;
+}
+
+uint32_t siridb_time_in_seconds(siridb_t * siridb, int64_t ts)
+{
+ return ts / siridb->time->factor;
+}
+
+uint64_t siridb_time_now(siridb_t * siridb, struct timespec now)
+{
+ return now.tv_sec * siridb->time->factor +
+ now.tv_nsec * (siridb->time->factor / 1000000000.0);
+}
--- /dev/null
+/*
+ * user.c - Contains functions for a SiriDB database user.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/access.h>
+#include <siri/db/db.h>
+#include <siri/db/user.h>
+#include <siri/db/users.h>
+#include <siri/err.h>
+#include <siri/grammar/grammar.h>
+#include <xstr/xstr.h>
+#include <owcrypt/owcrypt.h>
+#include <string.h>
+
+#define SIRIDB_MIN_PASSWORD_LEN 4
+#define SIRIDB_MAX_PASSWORD_LEN 128
+#define SIRIDB_MIN_USER_LEN 2
+#define SIRIDB_MAX_USER_LEN 60
+
+
+/*
+ * Creates a new user with reference counter zero.
+ *
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+siridb_user_t * siridb_user_new(void)
+{
+ siridb_user_t * user = malloc(sizeof(siridb_user_t));
+ if (user == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ user->access_bit = 0;
+ user->password = NULL;
+ user->name = NULL;
+ user->ref = 1;
+ }
+ return user;
+}
+
+/*
+ * This function can raise a SIGNAL. In this case the packer is not filled
+ * with the correct values.
+ */
+void siridb_user_prop(siridb_user_t * user, qp_packer_t * packer, int prop)
+{
+ switch (prop)
+ {
+ case CLERI_GID_K_NAME:
+ qp_add_string(packer, user->name);
+ break;
+ case CLERI_GID_K_ACCESS:
+ {
+ char buffer[SIRIDB_ACCESS_STR_MAX];
+ siridb_access_to_str(buffer, user->access_bit);
+ qp_add_string(packer, buffer);
+ }
+ break;
+ }
+}
+
+/*
+ * Returns 0 when successful or -1 if not.
+ *
+ * This function can have raised a SIGNAL when the result is -1 in case of
+ * a memory allocation error.
+ */
+int siridb_user_set_password(
+ siridb_user_t * user,
+ const char * password,
+ char * err_msg)
+{
+ char salt[OWCRYPT_SALT_SZ];
+ char encrypted[OWCRYPT_SZ];
+
+ if (strlen(password) < SIRIDB_MIN_PASSWORD_LEN)
+ {
+ if (err_msg != NULL)
+ {
+ sprintf(err_msg,
+ "Password should be at least %d characters.",
+ SIRIDB_MIN_PASSWORD_LEN);
+ }
+ return -1;
+ }
+
+ if (strlen(password) > SIRIDB_MAX_PASSWORD_LEN)
+ {
+ if (err_msg != NULL)
+ {
+ sprintf(err_msg,
+ "Password should be at most %d characters.",
+ SIRIDB_MAX_PASSWORD_LEN);
+ }
+ return -1;
+ }
+
+ if (!xstr_is_graph(password))
+ {
+ if (err_msg != NULL)
+ {
+ sprintf(err_msg,
+ "Password contains illegal characters. (only graphical "
+ "characters are allowed, no spaces, tabs etc.)");
+ }
+ return -1;
+ }
+
+ /* generate a random salt */
+ owcrypt_gen_salt(salt);
+
+ /* encrypt the users password */
+ owcrypt(password, salt, encrypted);
+
+ /* replace user password with encrypted password */
+ free(user->password);
+ user->password = strdup(encrypted);
+ if (user->password == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns 0 when successful or a positive value in case the name is not valid.
+ * A negative value is returned and a signal is raised in case a critical error
+ * has occurred.
+ *
+ * (err_msg is set in case of all errors)
+ */
+int siridb_user_set_name(
+ siridb_t * siridb,
+ siridb_user_t * user,
+ const char * name,
+ char * err_msg)
+{
+ if (strlen(name) < SIRIDB_MIN_USER_LEN)
+ {
+ sprintf(err_msg, "User name should be at least %d characters.",
+ SIRIDB_MIN_USER_LEN);
+ return 1;
+ }
+
+ if (strlen(name) > SIRIDB_MAX_USER_LEN)
+ {
+ sprintf(err_msg, "User name should be at least %d characters.",
+ SIRIDB_MAX_USER_LEN);
+ return 1;
+ }
+
+ if (!xstr_is_graph(name))
+ {
+ sprintf(err_msg,
+ "User name contains illegal characters. (only graphical "
+ "characters are allowed, no spaces, tabs etc.)");
+ return 1;
+ }
+
+ if (siridb_users_get_user(siridb, name, NULL) != NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "User '%s' already exists.",
+ name);
+ return 1;
+ }
+
+ free(user->name);
+ user->name = strdup(name);
+
+ if (user->name == NULL)
+ {
+ ERR_ALLOC
+ sprintf(err_msg, "Memory allocation error.");
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns true (1) if the user has access or false (0) if not. In case the
+ * user has no access, 'err_msg' is filled with an appropriate error message.
+ *
+ * Make sure 'err_msg' is a pointer to a string which can hold at least
+ * SIRIDB_MAX_SIZE_ERR_MSG.
+ */
+int siridb_user_check_access(
+ siridb_user_t * user,
+ uint32_t access_bit,
+ char * err_msg)
+{
+ if ((user->access_bit & access_bit) == access_bit)
+ {
+ return 1; /* true */
+ }
+
+ char buffer[SIRIDB_ACCESS_STR_MAX];
+ siridb_access_to_str(buffer, access_bit);
+ snprintf(
+ err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Access denied. User '%s' has no '%s' privileges.",
+ user->name,
+ buffer);
+
+ return 0; /* false */
+}
+
+int siridb_user_cexpr_cb(siridb_user_t * user, cexpr_condition_t * cond)
+{
+ /* str is not used for user */
+
+ switch (cond->prop)
+ {
+ case CLERI_GID_K_ACCESS:
+ return cexpr_int_cmp(cond->operator, user->access_bit, cond->int64);
+ case CLERI_GID_K_NAME:
+ return cexpr_str_cmp(cond->operator, user->name, cond->str);
+ }
+
+ /* this must NEVER happen */
+ log_critical("Unknown user property received: %d", cond->prop);
+ assert (0);
+ return -1;
+}
+
+/*
+ * Never call this function but use siridb_user_decref.
+ *
+ * Destroy user. (parsing NULL is not allowed)
+ */
+void siridb__user_free(siridb_user_t * user)
+{
+ free(user->name);
+ free(user->password);
+ free(user);
+}
--- /dev/null
+/*
+ * users.c - Collection of database users.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <logger/logger.h>
+#include <qpack/qpack.h>
+#include <siri/db/query.h>
+#include <siri/db/users.h>
+#include <siri/db/misc.h>
+#include <siri/err.h>
+#include <stdlib.h>
+#include <xstr/xstr.h>
+#include <string.h>
+#include <time.h>
+#include <xpath/xpath.h>
+#include <owcrypt/owcrypt.h>
+#include <siri/db/access.h>
+#include <base64/base64.h>
+
+
+#ifndef __APPLE__
+/* Required for compatibility with version < 2.0.14 */
+#include <crypt.h>
+#endif
+
+#define SIRIDB_USERS_SCHEMA 1
+#define SIRIDB_USERS_FN "users.dat"
+
+static inline int USERS_cmp(siridb_user_t * user, const char * name);
+static int USERS_free(siridb_user_t * user, void * args);
+static int USERS_save(siridb_user_t * user, qp_fpacker_t * fpacker);
+
+#define MSG_ERR_CANNOT_WRITE_USERS "Could not write users to file!"
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * (a SIGNAL might be raised in case of an error)
+ */
+int siridb_users_load(siridb_t * siridb)
+{
+ qp_unpacker_t * unpacker;
+ qp_obj_t qp_name;
+ qp_obj_t qp_password;
+ qp_obj_t qp_access_bit;
+ siridb_user_t * user;
+ char err_msg[SIRIDB_MAX_SIZE_ERR_MSG];
+
+ log_info("Loading users");
+
+ /* we should not have any users at this moment */
+ assert(siridb->users == NULL);
+
+ /* create a new user list */
+ siridb->users = llist_new();
+ if (siridb->users == NULL)
+ {
+ return -1;
+ }
+
+ /* get user access file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_USERS_FN)
+
+ if (!xpath_file_exist(fn))
+ {
+ /* we do not have a user access file, lets create the first user */
+ user = siridb_user_new();
+ if (user == NULL)
+ {
+ return -1; /* signal is raised */
+ }
+
+ user->access_bit = SIRIDB_ACCESS_PROFILE_FULL;
+
+ if ( siridb_user_set_name(siridb, user, "iris", err_msg) ||
+ siridb_user_set_password(user, "siri", err_msg) ||
+ siridb_users_add_user(siridb, user, err_msg))
+ {
+ log_error("%s", err_msg);
+ siridb_user_decref(user);
+ return -1;
+ }
+
+ return 0;
+ }
+
+ if ((unpacker = qp_unpacker_ff(fn)) == NULL)
+ {
+ return -1; /* a signal is raised is case of a memory error */
+ }
+
+ /* unpacker will be freed in case macro fails */
+ siridb_misc_schema_check(SIRIDB_USERS_SCHEMA)
+
+ int rc = 0;
+ while (qp_is_array(qp_next(unpacker, NULL)) &&
+ qp_next(unpacker, &qp_name) == QP_RAW &&
+ qp_next(unpacker, &qp_password) == QP_RAW &&
+ qp_password.len > 12 && /* old and new passwords are > 12 */
+ qp_next(unpacker, &qp_access_bit) == QP_INT64)
+ {
+ user = siridb_user_new();
+ if (user == NULL)
+ {
+ rc = -1; /* signal is raised */
+ }
+ else
+ {
+ user->name = strndup((char *) qp_name.via.raw, qp_name.len);
+ user->password = strndup(
+ (char *) qp_password.via.raw, qp_password.len);
+
+ if (user->name == NULL || user->password == NULL)
+ {
+ ERR_ALLOC
+ siridb_user_decref(user);
+ rc = -1;
+ }
+ else
+ {
+ user->access_bit = (uint32_t) qp_access_bit.via.int64;
+ if (llist_append(siridb->users, user))
+ {
+ siridb_user_decref(user);
+ ERR_ALLOC
+ rc = -1;
+ }
+ }
+ }
+ }
+
+ /* free unpacker */
+ qp_unpacker_ff_free(unpacker);
+
+ return rc;
+}
+
+/*
+ * Typedef: sirinet_clserver_get_file
+ *
+ * Returns the length of the content for a file and set buffer with the file
+ * content. Note that malloc is used to allocate memory for the buffer.
+ *
+ * In case of an error -1 is returned and buffer will be set to NULL.
+ */
+ssize_t siridb_users_get_file(char ** buffer, siridb_t * siridb)
+{
+ /* get users file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_USERS_FN)
+
+ return xpath_get_content(buffer, fn);
+}
+
+/*
+ * Destroy servers, parsing NULL is not allowed.
+ */
+void siridb_users_free(llist_t * users)
+{
+ llist_free_cb(users, (llist_cb) USERS_free, NULL);
+}
+
+/*
+ * Returns 0 when successful,or -1 is returned in case of a critical
+ * error. (a critical error also raises a signal). The err_msg will contain
+ * the error in any case.
+ */
+int siridb_users_add_user(
+ siridb_t * siridb,
+ siridb_user_t * user,
+ char * err_msg)
+{
+ /* add the user to the users */
+ if (llist_append(siridb->users, user))
+ {
+ /* this is critical, a signal is raised */
+ ERR_ALLOC
+ sprintf(err_msg, "Memory allocation error.");
+ return -1;
+ }
+
+ if (siridb_users_save(siridb))
+ {
+ /* this is critical, a signal is raised */
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Could not save user '%s' to file.",
+ user->name);
+ log_critical(err_msg);
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns NULL when the user is not found or when the given password is
+ * incorrect. When *password is NULL the password will NOT be checked and
+ * the user will be returned when found.
+ */
+siridb_user_t * siridb_users_get_user(
+ siridb_t * siridb,
+ const char * name,
+ const char * password)
+{
+ llist_t * users = siridb->users;
+ siridb_user_t * user;
+ char pw[OWCRYPT_SZ];
+
+#ifndef __APPLE__
+ /* Required for compatibility with version < 2.0.14 */
+ char * fallback_pw;
+ struct crypt_data fallback_data;
+#endif
+
+ if ((user = llist_get(
+ users,
+ (llist_cb) USERS_cmp,
+ (void *) name)) == NULL)
+ {
+ return NULL;
+ }
+
+ if (password == NULL)
+ {
+ return user;
+ }
+
+ owcrypt(password, user->password, pw);
+ if (strcmp(pw, user->password) == 0)
+ {
+ return user;
+ }
+#ifndef __APPLE__
+ /* Required for compatibility with version < 2.0.14 */
+ else if (user->password[0] == '$')
+ {
+ /* this will migrate as soon as a user logs in */
+ _Bool is_valid;
+ fallback_data.initialized = 0;
+ fallback_pw = crypt_r(password, user->password, &fallback_data);
+ is_valid = strcmp(fallback_pw, user->password) == 0;
+ (void) (is_valid && \
+ siridb_user_set_password(user, password, NULL) == 0 && \
+ siridb_users_save(siridb));
+ return is_valid ? user : NULL;
+ }
+#endif
+ return NULL;
+}
+
+siridb_user_t * siridb_users_get_user_from_basic(
+ siridb_t * siridb,
+ const char * data,
+ size_t n)
+{
+ size_t size, nn, end;
+ char * b64 = base64_decode(data, n, &size);
+ siridb_user_t * user = NULL;
+
+ for (nn = 0, end = size; nn < end; ++nn)
+ {
+ if (b64[nn] == ':')
+ {
+ b64[nn] = '\0';
+
+ if (++nn > end)
+ break;
+
+ user = siridb_users_get_user(siridb, b64, b64 + nn);
+ break;
+ }
+ }
+
+ free(b64);
+ return user;
+}
+
+/*
+ * We get and remove the user in one code block so we do not need a dropped
+ * flag on the user object.
+ *
+ * Returns 0 if successful. In case of an error -1 is returned and err_msg
+ * is set to an appropriate value.
+ */
+int siridb_users_drop_user(
+ siridb_t * siridb,
+ const char * name,
+ char * err_msg)
+{
+ siridb_user_t * user;
+
+ if ((user = llist_remove(
+ siridb->users,
+ (llist_cb) USERS_cmp,
+ (void *) name)) == NULL)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "User '%s' does not exist.",
+ name);
+ return -1;
+ }
+
+ /* decrement reference for user object */
+ siridb_user_decref(user);
+
+ if (siridb_users_save(siridb))
+ {
+ log_critical(MSG_ERR_CANNOT_WRITE_USERS);
+ sprintf(err_msg, MSG_ERR_CANNOT_WRITE_USERS);
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns 0 if successful; EOF and a signal is raised in case an error occurred.
+ */
+int siridb_users_save(siridb_t * siridb)
+{
+ qp_fpacker_t * fpacker;
+
+ /* get user access file name */
+ siridb_misc_get_fn(fn, siridb->dbpath, SIRIDB_USERS_FN)
+
+ if (
+ /* open a new user file */
+ (fpacker = qp_open(fn, "w")) == NULL ||
+
+ /* open a new array */
+ qp_fadd_type(fpacker, QP_ARRAY_OPEN) ||
+
+ /* write the current schema */
+ qp_fadd_int64(fpacker, SIRIDB_USERS_SCHEMA) ||
+
+ /* we can and should skip this if we have no users to save */
+ llist_walk(siridb->users, (llist_cb) USERS_save, fpacker) ||
+
+ /* close file pointer */
+ qp_close(fpacker))
+ {
+ ERR_FILE
+ return EOF;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns 0 if successful and -1 in case an error occurred.
+ */
+static int USERS_save(siridb_user_t * user, qp_fpacker_t * fpacker)
+{
+ int rc = 0;
+
+ rc += qp_fadd_type(fpacker, QP_ARRAY3);
+ rc += qp_fadd_string(fpacker, user->name);
+ rc += qp_fadd_string(fpacker, user->password);
+ rc += qp_fadd_int64(fpacker, (int64_t) user->access_bit);
+ return rc;
+}
+
+static inline int USERS_cmp(siridb_user_t * user, const char * name)
+{
+ return (strcmp(user->name, name) == 0);
+}
+
+static int USERS_free(
+ siridb_user_t * user,
+ void * args __attribute__((unused)))
+{
+ siridb_user_decref(user);
+ return 0;
+}
+
--- /dev/null
+/*
+ * variance.c - Calculate variance for points.
+ */
+#include <assert.h>
+#include <math.h>
+#include <siri/db/points.h>
+#include <siri/db/variance.h>
+
+double siridb_variance(siridb_points_t * points)
+{
+ double mean = 0.0;
+ double variance = 0.0;
+ size_t i;
+
+ switch (points->tp)
+ {
+ case TP_INT:
+ for (i = 0; i < points->len; i++)
+ {
+ mean += (points->data + i)->val.int64;
+ }
+
+ mean /= points->len;
+
+ for (i = 0; i < points->len; i++)
+ {
+ variance += pow(
+ (double) (points->data + i)->val.int64 - mean,
+ 2);
+ }
+ break;\
+ case TP_DOUBLE:
+ for (i = 0; i < points->len; i++)
+ {
+ mean += (points->data + i)->val.real;
+ }
+
+ mean /= points->len;
+
+ for (i = 0; i < points->len; i++)
+ {
+ variance += pow(
+ (points->data + i)->val.real - mean,
+ 2);
+ }
+ break;
+ default:
+ assert (0);
+ break;
+ }
+
+ return variance;
+}
--- /dev/null
+/*
+ * walker.c - Creates enter and exit nodes.
+ */
+#include <siri/db/walker.h>
+#include <siri/err.h>
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+siridb_walker_t * siridb_walker_new(
+ siridb_t * siridb,
+ const uint64_t now,
+ uint8_t * flags)
+{
+ siridb_walker_t * walker = malloc(sizeof(siridb_walker_t));
+ if (walker == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ walker->siridb = siridb;
+ walker->now = now;
+ walker->flags = flags;
+ walker->start = NULL;
+ walker->enter_nodes = NULL;
+ walker->exit_nodes = NULL;
+ }
+ return walker;
+}
+
+/*
+ * Destroy walker. (parsing NULL is NOT allowed)
+ */
+siridb_nodes_t * siridb_walker_free(siridb_walker_t * walker)
+{
+ siridb_nodes_t * nodes;
+ if (walker->start == NULL)
+ {
+ nodes = walker->exit_nodes;
+ }
+ else
+ {
+ walker->enter_nodes->next = walker->exit_nodes;
+ nodes = walker->start;
+ }
+ free(walker);
+ return nodes;
+}
+
+/*
+ * Returns 0 if successful and -1 in case of an error.
+ * (signal is set in case of an error)
+ */
+int siridb_walker_append(
+ siridb_walker_t * walker,
+ cleri_node_t * node,
+ uv_async_cb cb)
+{
+ siridb_nodes_t * wnode = malloc(sizeof(siridb_nodes_t));
+ if (wnode == NULL)
+ {
+ ERR_ALLOC
+ return -1;
+ }
+ wnode->node = node;
+ wnode->cb = cb;
+ wnode->next = NULL;
+
+ if (walker->start == NULL)
+ {
+ walker->start = wnode;
+ walker->enter_nodes = wnode;
+ }
+ else
+ {
+ walker->enter_nodes->next = wnode;
+ walker->enter_nodes = wnode;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns 0 if successful and -1 in case of an error.
+ * (signal is set in case of an error)
+ */
+int siridb_walker_insert(
+ siridb_walker_t * walker,
+ cleri_node_t * node,
+ uv_async_cb cb)
+{
+ siridb_nodes_t * current = walker->exit_nodes;
+
+ walker->exit_nodes = malloc(sizeof(siridb_nodes_t));
+ if (walker->exit_nodes == NULL)
+ {
+ ERR_ALLOC
+ /* restore walker node */
+ walker->exit_nodes = current;
+ return -1;
+ }
+ walker->exit_nodes->node = node;
+ walker->exit_nodes->cb = cb;
+ walker->exit_nodes->next = current;
+
+ return 0;
+}
--- /dev/null
+/*
+ * err.c - SiriDB Error.
+ */
+#include <siri/err.h>
+
+int siri_err = 0;
--- /dev/null
+/*
+ * siri/evars.c
+ */
+
+#include <siri/evars.h>
+#include <siri/net/tcp.h>
+
+static void evars__bool(const char * evar, uint8_t * b)
+{
+ char * u8str = getenv(evar);
+ if (!u8str)
+ return;
+
+ *b = (_Bool) strtoul(u8str, NULL, 10);
+}
+
+static void evars__u16(const char * evar, uint16_t * u16)
+{
+ char * u16str = getenv(evar);
+ if (!u16str)
+ return;
+
+ *u16 = (uint16_t) strtoul(u16str, NULL, 10);
+}
+
+static void evars__u16_mm(
+ const char * evar,
+ uint16_t * u16,
+ uint16_t mi,
+ uint16_t ma)
+{
+ uint16_t val;
+ char * u16str = getenv(evar);
+ if (!u16str)
+ return;
+
+ val = (uint16_t) strtoul(u16str, NULL, 10);
+ if (val < mi || val > ma)
+ return;
+
+ *u16 = val;
+}
+
+static void evars__u32_mm(
+ const char * evar,
+ uint32_t * u32,
+ uint32_t mi,
+ uint32_t ma)
+{
+ uint32_t val;
+ char * u32str = getenv(evar);
+ if (!u32str)
+ return;
+
+ val = (uint32_t) strtoull(u32str, NULL, 10);
+ if (val < mi || val > ma)
+ return;
+
+ *u32 = val;
+}
+
+static void evars__to_strn(const char * evar, char * s, size_t n)
+{
+ char * str = getenv(evar);
+ if (!str || strlen(str) >= n)
+ return;
+
+ strncpy(s, str, n);
+}
+
+static void evars__ip_support(const char * evar, uint8_t * ip_support)
+{
+ char * str = getenv(evar);
+ if (!str)
+ return;
+
+ if (strcasecmp(str, "ALL") == 0)
+ {
+ *ip_support = IP_SUPPORT_ALL;
+ }
+ else if (strcasecmp(str, "IPV4ONLY") == 0)
+ {
+ *ip_support = IP_SUPPORT_IPV4ONLY;
+ }
+ else if (strcasecmp(str, "IPV6ONLY") == 0)
+ {
+ *ip_support = IP_SUPPORT_IPV6ONLY;
+ }
+}
+
+static void evars__to_addr(const char * evar, char ** addr)
+{
+ struct in_addr sa;
+ struct in6_addr sa6;
+
+ char * str = getenv(evar);
+ if (!str || (
+ !inet_pton(AF_INET, str, &sa) &&
+ !inet_pton(AF_INET6, str, &sa6)))
+ return;
+
+ str = strdup(str);
+ if (!str)
+ return;
+
+ free(*addr);
+ *addr = str;
+}
+
+static void evars__to_addr_port(
+ const char * evar,
+ char * addr,
+ uint16_t * port)
+{
+ char * str = getenv(evar);
+ if (!str)
+ return;
+
+ (void) sirinet_extract_addr_port(str, addr, port);
+}
+
+
+void siri_evars_parse(siri_t * siri)
+{
+ evars__u16(
+ "SIRIDB_LISTEN_CLIENT_PORT",
+ &siri->cfg->listen_client_port);
+ evars__u16(
+ "SIRIDB_HTTP_STATUS_PORT",
+ &siri->cfg->http_status_port);
+ evars__u16(
+ "SIRIDB_HTTP_API_PORT",
+ &siri->cfg->http_api_port);
+ evars__u16(
+ "SIRIDB_MAX_OPEN_FILES",
+ &siri->cfg->max_open_files);
+ evars__bool(
+ "SIRIDB_ENABLE_PIPE_SUPPORT",
+ &siri->cfg->pipe_support);
+ evars__bool(
+ "SIRIDB_ENABLE_SHARD_COMPRESSION",
+ &siri->cfg->shard_compression);
+ evars__bool(
+ "SIRIDB_ENABLE_SHARD_AUTO_DURATION",
+ &siri->cfg->shard_auto_duration);
+ evars__to_strn(
+ "SIRIDB_DB_PATH",
+ siri->cfg->db_path,
+ sizeof(siri->cfg->db_path));
+ /* Read old environment variable for backwards compatibility */
+ evars__to_strn(
+ "SIRIDB_DEFAULT_DB_PATH",
+ siri->cfg->db_path,
+ sizeof(siri->cfg->db_path));
+ evars__u32_mm(
+ "SIRIDB_BUFFER_SYNC_INTERVAL",
+ &siri->cfg->buffer_sync_interval,
+ 0, 300000);
+ evars__u16_mm(
+ "SIRIDB_HEARTBEAT_INTERVAL",
+ &siri->cfg->heartbeat_interval,
+ 3, 300);
+ evars__u32_mm(
+ "SIRIDB_OPTIMIZING_INTERVAL",
+ &siri->cfg->optimize_interval,
+ 0, 2419200);
+ evars__ip_support(
+ "SIRIDB_IP_SUPPORT",
+ &siri->cfg->ip_support);
+ evars__to_addr(
+ "SIRIDB_BIND_CLIENT_ADDRESS",
+ &siri->cfg->bind_client_addr);
+ evars__to_addr(
+ "SIRIDB_BIND_SERVER_ADDRESS",
+ &siri->cfg->bind_backend_addr);
+ evars__to_strn(
+ "SIRIDB_PIPE_CLIENT_NAME",
+ siri->cfg->pipe_client_name,
+ sizeof(siri->cfg->pipe_client_name)-2);
+ evars__to_addr_port(
+ "SIRIDB_SERVER_NAME",
+ siri->cfg->server_address,
+ &siri->cfg->listen_backend_port);
+}
--- /dev/null
+/*
+ * handler.c - File handler for shard files.
+ */
+#include <logger/logger.h>
+#include <siri/err.h>
+#include <siri/file/handler.h>
+#include <stdlib.h>
+
+siri_fh_t * siri_fh_new(uint16_t size)
+{
+ siri_fh_t * fh = malloc(sizeof(siri_fh_t));
+ if (fh == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ fh->size = size;
+ fh->idx = 0;
+ fh->fpointers = calloc(size, sizeof(siri_fp_t *));
+ if (fh->fpointers == NULL)
+ {
+ ERR_ALLOC
+ free(fh);
+ fh = NULL;
+ }
+ }
+ return fh;
+}
+
+/*
+ * Destroy file handler. (closes all open files in the file handler)
+ * In case closing an open file fails, a SIGNAL will be raised.
+ */
+void siri_fh_free(siri_fh_t * fh)
+{
+ siri_fp_t ** fp;
+ uint16_t i;
+
+ if (fh == NULL)
+ {
+ return;
+ }
+
+ for (i = 0; i < fh->size; i++)
+ {
+ fp = fh->fpointers + i;
+
+ if (*fp == NULL)
+ {
+ break;
+ }
+ siri_fp_decref(*fp);
+ }
+ free(fh->fpointers);
+ free(fh);
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ */
+int siri_fopen(
+ siri_fh_t * fh,
+ siri_fp_t * fp,
+ const char * fn,
+ const char * modes)
+{
+ siri_fp_t ** dest = fh->fpointers + fh->idx;
+
+ /* close and possible free file pointer at next position */
+ if (*dest != NULL)
+ {
+ siri_fp_decref(*dest);
+ }
+
+ /* assign file pointer */
+ *dest = fp;
+
+ /* increment reference counter (must be done even if open fails) */
+ fp->ref++;
+
+ if ((fp->fp = fopen(fn, modes)) == NULL)
+ {
+ log_critical("Cannot open file: '%s' using mode '%s'", fn, modes);
+ return -1;
+ }
+
+ /* set file handler pointer to next position */
+ fh->idx = (fh->idx + 1) % fh->size;
+
+ return 0;
+}
+
+
--- /dev/null
+/*
+ * pointer.c - File pointer used in combination with file handler.
+ */
+#include <logger/logger.h>
+#include <siri/err.h>
+#include <siri/file/pointer.h>
+#include <stdlib.h>
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+siri_fp_t * siri_fp_new(void)
+{
+ siri_fp_t * fp = malloc(sizeof(siri_fp_t));
+ if (fp == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ fp->fp = NULL;
+ fp->ref = 1;
+ }
+ return fp;
+}
+
+/*
+ * This function will always close the file when open but only frees the
+ * object from memory when reference count 0 is reached.
+ *
+ * When an error occurs while closing the file, a SIGNAL is raised.
+ */
+void siri_fp_decref(siri_fp_t * fp)
+{
+ if (fp->fp != NULL)
+ {
+ if (fclose(fp->fp))
+ {
+ ERR_FILE
+ }
+ fp->fp = NULL;
+ }
+ if (!--fp->ref)
+ {
+ free(fp);
+ }
+}
+
+/*
+ * This will close a file and a SIGNAL is raised is an error occurs.
+ */
+void siri_fp_close(siri_fp_t * fp)
+{
+ if (fp->fp != NULL)
+ {
+ if (fclose(fp->fp))
+ {
+ ERR_FILE
+ }
+ fp->fp = NULL;
+ }
+}
--- /dev/null
+/*
+ * siri/grammar/grammar.c
+ *
+ * This grammar is generated using the Grammar.export_c() method and
+ * should be used with the libcleri module.
+ *
+ * Source class: SiriGrammar
+ * Created at: 2020-09-25 10:57:26
+ */
+
+#include "siri/grammar/grammar.h"
+#include <stdio.h>
+
+#define CLERI_CASE_SENSITIVE 0
+#define CLERI_CASE_INSENSITIVE 1
+
+#define CLERI_FIRST_MATCH 0
+#define CLERI_MOST_GREEDY 1
+
+cleri_grammar_t * compile_siri_grammar_grammar(void)
+{
+ cleri_t * r_float = cleri_regex(CLERI_GID_R_FLOAT, "^[-+]?[0-9]*\\.?[0-9]+");
+ cleri_t * r_integer = cleri_regex(CLERI_GID_R_INTEGER, "^[-+]?[0-9]+");
+ cleri_t * r_uinteger = cleri_regex(CLERI_GID_R_UINTEGER, "^[0-9]+");
+ cleri_t * r_time_str = cleri_regex(CLERI_GID_R_TIME_STR, "^[0-9]+[smhdw]");
+ cleri_t * r_singleq_str = cleri_regex(CLERI_GID_R_SINGLEQ_STR, "^(?:\'(?:[^\']*)\')+");
+ cleri_t * r_doubleq_str = cleri_regex(CLERI_GID_R_DOUBLEQ_STR, "^(?:\"(?:[^\"]*)\")+");
+ cleri_t * r_grave_str = cleri_regex(CLERI_GID_R_GRAVE_STR, "^(?:`(?:[^`]*)`)+");
+ cleri_t * r_uuid_str = cleri_regex(CLERI_GID_R_UUID_STR, "^[0-9a-f]{8}\\-[0-9a-f]{4}\\-[0-9a-f]{4}\\-[0-9a-f]{4}\\-[0-9a-f]{12}");
+ cleri_t * r_regex = cleri_regex(CLERI_GID_R_REGEX, "^(/[^/\\\\]*(?:\\\\.[^/\\\\]*)*/i?)");
+ cleri_t * r_comment = cleri_regex(CLERI_GID_R_COMMENT, "^#.*");
+ cleri_t * k_access = cleri_keyword(CLERI_GID_K_ACCESS, "access", CLERI_CASE_SENSITIVE);
+ cleri_t * k_active_handles = cleri_keyword(CLERI_GID_K_ACTIVE_HANDLES, "active_handles", CLERI_CASE_SENSITIVE);
+ cleri_t * k_active_tasks = cleri_keyword(CLERI_GID_K_ACTIVE_TASKS, "active_tasks", CLERI_CASE_SENSITIVE);
+ cleri_t * k_address = cleri_keyword(CLERI_GID_K_ADDRESS, "address", CLERI_CASE_SENSITIVE);
+ cleri_t * k_after = cleri_keyword(CLERI_GID_K_AFTER, "after", CLERI_CASE_SENSITIVE);
+ cleri_t * k_all = cleri_keyword(CLERI_GID_K_ALL, "all", CLERI_CASE_SENSITIVE);
+ cleri_t * k_alter = cleri_keyword(CLERI_GID_K_ALTER, "alter", CLERI_CASE_SENSITIVE);
+ cleri_t * k_and = cleri_keyword(CLERI_GID_K_AND, "and", CLERI_CASE_SENSITIVE);
+ cleri_t * k_as = cleri_keyword(CLERI_GID_K_AS, "as", CLERI_CASE_SENSITIVE);
+ cleri_t * k_backup_mode = cleri_keyword(CLERI_GID_K_BACKUP_MODE, "backup_mode", CLERI_CASE_SENSITIVE);
+ cleri_t * k_before = cleri_keyword(CLERI_GID_K_BEFORE, "before", CLERI_CASE_SENSITIVE);
+ cleri_t * k_buffer_size = cleri_keyword(CLERI_GID_K_BUFFER_SIZE, "buffer_size", CLERI_CASE_SENSITIVE);
+ cleri_t * k_buffer_path = cleri_keyword(CLERI_GID_K_BUFFER_PATH, "buffer_path", CLERI_CASE_SENSITIVE);
+ cleri_t * k_between = cleri_keyword(CLERI_GID_K_BETWEEN, "between", CLERI_CASE_SENSITIVE);
+ cleri_t * k_count = cleri_keyword(CLERI_GID_K_COUNT, "count", CLERI_CASE_SENSITIVE);
+ cleri_t * k_create = cleri_keyword(CLERI_GID_K_CREATE, "create", CLERI_CASE_SENSITIVE);
+ cleri_t * k_critical = cleri_keyword(CLERI_GID_K_CRITICAL, "critical", CLERI_CASE_SENSITIVE);
+ cleri_t * k_database = cleri_keyword(CLERI_GID_K_DATABASE, "database", CLERI_CASE_SENSITIVE);
+ cleri_t * k_dbname = cleri_keyword(CLERI_GID_K_DBNAME, "dbname", CLERI_CASE_SENSITIVE);
+ cleri_t * k_dbpath = cleri_keyword(CLERI_GID_K_DBPATH, "dbpath", CLERI_CASE_SENSITIVE);
+ cleri_t * k_debug = cleri_keyword(CLERI_GID_K_DEBUG, "debug", CLERI_CASE_SENSITIVE);
+ cleri_t * k_derivative = cleri_keyword(CLERI_GID_K_DERIVATIVE, "derivative", CLERI_CASE_SENSITIVE);
+ cleri_t * k_difference = cleri_keyword(CLERI_GID_K_DIFFERENCE, "difference", CLERI_CASE_SENSITIVE);
+ cleri_t * k_drop = cleri_keyword(CLERI_GID_K_DROP, "drop", CLERI_CASE_SENSITIVE);
+ cleri_t * k_drop_threshold = cleri_keyword(CLERI_GID_K_DROP_THRESHOLD, "drop_threshold", CLERI_CASE_SENSITIVE);
+ cleri_t * k_duration_log = cleri_keyword(CLERI_GID_K_DURATION_LOG, "duration_log", CLERI_CASE_SENSITIVE);
+ cleri_t * k_duration_num = cleri_keyword(CLERI_GID_K_DURATION_NUM, "duration_num", CLERI_CASE_SENSITIVE);
+ cleri_t * k_end = cleri_keyword(CLERI_GID_K_END, "end", CLERI_CASE_SENSITIVE);
+ cleri_t * k_error = cleri_keyword(CLERI_GID_K_ERROR, "error", CLERI_CASE_SENSITIVE);
+ cleri_t * k_expiration_log = cleri_keyword(CLERI_GID_K_EXPIRATION_LOG, "expiration_log", CLERI_CASE_SENSITIVE);
+ cleri_t * k_expiration_num = cleri_keyword(CLERI_GID_K_EXPIRATION_NUM, "expiration_num", CLERI_CASE_SENSITIVE);
+ cleri_t * k_expression = cleri_keyword(CLERI_GID_K_EXPRESSION, "expression", CLERI_CASE_SENSITIVE);
+ cleri_t * k_false = cleri_keyword(CLERI_GID_K_FALSE, "false", CLERI_CASE_SENSITIVE);
+ cleri_t * k_fifo_files = cleri_keyword(CLERI_GID_K_FIFO_FILES, "fifo_files", CLERI_CASE_SENSITIVE);
+ cleri_t * k_filter = cleri_keyword(CLERI_GID_K_FILTER, "filter", CLERI_CASE_SENSITIVE);
+ cleri_t * k_first = cleri_keyword(CLERI_GID_K_FIRST, "first", CLERI_CASE_SENSITIVE);
+ cleri_t * k_float = cleri_keyword(CLERI_GID_K_FLOAT, "float", CLERI_CASE_SENSITIVE);
+ cleri_t * k_for = cleri_keyword(CLERI_GID_K_FOR, "for", CLERI_CASE_SENSITIVE);
+ cleri_t * k_from = cleri_keyword(CLERI_GID_K_FROM, "from", CLERI_CASE_SENSITIVE);
+ cleri_t * k_full = cleri_keyword(CLERI_GID_K_FULL, "full", CLERI_CASE_SENSITIVE);
+ cleri_t * k_grant = cleri_keyword(CLERI_GID_K_GRANT, "grant", CLERI_CASE_SENSITIVE);
+ cleri_t * k_group = cleri_keyword(CLERI_GID_K_GROUP, "group", CLERI_CASE_SENSITIVE);
+ cleri_t * k_groups = cleri_keyword(CLERI_GID_K_GROUPS, "groups", CLERI_CASE_SENSITIVE);
+ cleri_t * k_help = cleri_choice(
+ CLERI_GID_K_HELP,
+ CLERI_MOST_GREEDY,
+ 2,
+ cleri_keyword(CLERI_NONE, "help", CLERI_CASE_SENSITIVE),
+ cleri_token(CLERI_NONE, "?")
+ );
+ cleri_t * k_idle_percentage = cleri_keyword(CLERI_GID_K_IDLE_PERCENTAGE, "idle_percentage", CLERI_CASE_SENSITIVE);
+ cleri_t * k_idle_time = cleri_keyword(CLERI_GID_K_IDLE_TIME, "idle_time", CLERI_CASE_SENSITIVE);
+ cleri_t * k_inf = cleri_keyword(CLERI_GID_K_INF, "inf", CLERI_CASE_SENSITIVE);
+ cleri_t * k_info = cleri_keyword(CLERI_GID_K_INFO, "info", CLERI_CASE_SENSITIVE);
+ cleri_t * k_ignore_threshold = cleri_keyword(CLERI_GID_K_IGNORE_THRESHOLD, "ignore_threshold", CLERI_CASE_SENSITIVE);
+ cleri_t * k_insert = cleri_keyword(CLERI_GID_K_INSERT, "insert", CLERI_CASE_SENSITIVE);
+ cleri_t * k_integer = cleri_keyword(CLERI_GID_K_INTEGER, "integer", CLERI_CASE_SENSITIVE);
+ cleri_t * k_intersection = cleri_choice(
+ CLERI_GID_K_INTERSECTION,
+ CLERI_FIRST_MATCH,
+ 2,
+ cleri_token(CLERI_NONE, "&"),
+ cleri_keyword(CLERI_NONE, "intersection", CLERI_CASE_SENSITIVE)
+ );
+ cleri_t * k_interval = cleri_keyword(CLERI_GID_K_INTERVAL, "interval", CLERI_CASE_SENSITIVE);
+ cleri_t * k_ip_support = cleri_keyword(CLERI_GID_K_IP_SUPPORT, "ip_support", CLERI_CASE_SENSITIVE);
+ cleri_t * k_last = cleri_keyword(CLERI_GID_K_LAST, "last", CLERI_CASE_SENSITIVE);
+ cleri_t * k_length = cleri_keyword(CLERI_GID_K_LENGTH, "length", CLERI_CASE_SENSITIVE);
+ cleri_t * k_libuv = cleri_keyword(CLERI_GID_K_LIBUV, "libuv", CLERI_CASE_SENSITIVE);
+ cleri_t * k_limit = cleri_keyword(CLERI_GID_K_LIMIT, "limit", CLERI_CASE_SENSITIVE);
+ cleri_t * k_list = cleri_keyword(CLERI_GID_K_LIST, "list", CLERI_CASE_SENSITIVE);
+ cleri_t * k_list_limit = cleri_keyword(CLERI_GID_K_LIST_LIMIT, "list_limit", CLERI_CASE_SENSITIVE);
+ cleri_t * k_log = cleri_keyword(CLERI_GID_K_LOG, "log", CLERI_CASE_SENSITIVE);
+ cleri_t * k_log_level = cleri_keyword(CLERI_GID_K_LOG_LEVEL, "log_level", CLERI_CASE_SENSITIVE);
+ cleri_t * k_max = cleri_keyword(CLERI_GID_K_MAX, "max", CLERI_CASE_SENSITIVE);
+ cleri_t * k_max_open_files = cleri_keyword(CLERI_GID_K_MAX_OPEN_FILES, "max_open_files", CLERI_CASE_SENSITIVE);
+ cleri_t * k_mean = cleri_keyword(CLERI_GID_K_MEAN, "mean", CLERI_CASE_SENSITIVE);
+ cleri_t * k_median = cleri_keyword(CLERI_GID_K_MEDIAN, "median", CLERI_CASE_SENSITIVE);
+ cleri_t * k_median_high = cleri_keyword(CLERI_GID_K_MEDIAN_HIGH, "median_high", CLERI_CASE_SENSITIVE);
+ cleri_t * k_median_low = cleri_keyword(CLERI_GID_K_MEDIAN_LOW, "median_low", CLERI_CASE_SENSITIVE);
+ cleri_t * k_mem_usage = cleri_keyword(CLERI_GID_K_MEM_USAGE, "mem_usage", CLERI_CASE_SENSITIVE);
+ cleri_t * k_merge = cleri_keyword(CLERI_GID_K_MERGE, "merge", CLERI_CASE_SENSITIVE);
+ cleri_t * k_min = cleri_keyword(CLERI_GID_K_MIN, "min", CLERI_CASE_SENSITIVE);
+ cleri_t * k_modify = cleri_keyword(CLERI_GID_K_MODIFY, "modify", CLERI_CASE_SENSITIVE);
+ cleri_t * k_name = cleri_keyword(CLERI_GID_K_NAME, "name", CLERI_CASE_SENSITIVE);
+ cleri_t * k_nan = cleri_keyword(CLERI_GID_K_NAN, "nan", CLERI_CASE_SENSITIVE);
+ cleri_t * k_ninf = cleri_sequence(
+ CLERI_GID_K_NINF,
+ 2,
+ cleri_token(CLERI_NONE, "-"),
+ k_inf
+ );
+ cleri_t * k_now = cleri_keyword(CLERI_GID_K_NOW, "now", CLERI_CASE_SENSITIVE);
+ cleri_t * k_number = cleri_keyword(CLERI_GID_K_NUMBER, "number", CLERI_CASE_SENSITIVE);
+ cleri_t * k_online = cleri_keyword(CLERI_GID_K_ONLINE, "online", CLERI_CASE_SENSITIVE);
+ cleri_t * k_open_files = cleri_keyword(CLERI_GID_K_OPEN_FILES, "open_files", CLERI_CASE_SENSITIVE);
+ cleri_t * k_or = cleri_keyword(CLERI_GID_K_OR, "or", CLERI_CASE_SENSITIVE);
+ cleri_t * k_password = cleri_keyword(CLERI_GID_K_PASSWORD, "password", CLERI_CASE_SENSITIVE);
+ cleri_t * k_points = cleri_keyword(CLERI_GID_K_POINTS, "points", CLERI_CASE_SENSITIVE);
+ cleri_t * k_pool = cleri_keyword(CLERI_GID_K_POOL, "pool", CLERI_CASE_SENSITIVE);
+ cleri_t * k_pools = cleri_keyword(CLERI_GID_K_POOLS, "pools", CLERI_CASE_SENSITIVE);
+ cleri_t * k_port = cleri_keyword(CLERI_GID_K_PORT, "port", CLERI_CASE_SENSITIVE);
+ cleri_t * k_prefix = cleri_keyword(CLERI_GID_K_PREFIX, "prefix", CLERI_CASE_SENSITIVE);
+ cleri_t * k_pvariance = cleri_keyword(CLERI_GID_K_PVARIANCE, "pvariance", CLERI_CASE_SENSITIVE);
+ cleri_t * k_read = cleri_keyword(CLERI_GID_K_READ, "read", CLERI_CASE_SENSITIVE);
+ cleri_t * k_received_points = cleri_keyword(CLERI_GID_K_RECEIVED_POINTS, "received_points", CLERI_CASE_SENSITIVE);
+ cleri_t * k_reindex_progress = cleri_keyword(CLERI_GID_K_REINDEX_PROGRESS, "reindex_progress", CLERI_CASE_SENSITIVE);
+ cleri_t * k_revoke = cleri_keyword(CLERI_GID_K_REVOKE, "revoke", CLERI_CASE_SENSITIVE);
+ cleri_t * k_select = cleri_keyword(CLERI_GID_K_SELECT, "select", CLERI_CASE_SENSITIVE);
+ cleri_t * k_select_points_limit = cleri_keyword(CLERI_GID_K_SELECT_POINTS_LIMIT, "select_points_limit", CLERI_CASE_SENSITIVE);
+ cleri_t * k_selected_points = cleri_keyword(CLERI_GID_K_SELECTED_POINTS, "selected_points", CLERI_CASE_SENSITIVE);
+ cleri_t * k_series = cleri_keyword(CLERI_GID_K_SERIES, "series", CLERI_CASE_SENSITIVE);
+ cleri_t * k_server = cleri_keyword(CLERI_GID_K_SERVER, "server", CLERI_CASE_SENSITIVE);
+ cleri_t * k_servers = cleri_keyword(CLERI_GID_K_SERVERS, "servers", CLERI_CASE_SENSITIVE);
+ cleri_t * k_set = cleri_keyword(CLERI_GID_K_SET, "set", CLERI_CASE_SENSITIVE);
+ cleri_t * k_shard_duration = cleri_keyword(CLERI_GID_K_SHARD_DURATION, "shard_duration", CLERI_CASE_SENSITIVE);
+ cleri_t * k_shards = cleri_keyword(CLERI_GID_K_SHARDS, "shards", CLERI_CASE_SENSITIVE);
+ cleri_t * k_show = cleri_keyword(CLERI_GID_K_SHOW, "show", CLERI_CASE_SENSITIVE);
+ cleri_t * k_sid = cleri_keyword(CLERI_GID_K_SID, "sid", CLERI_CASE_SENSITIVE);
+ cleri_t * k_size = cleri_keyword(CLERI_GID_K_SIZE, "size", CLERI_CASE_SENSITIVE);
+ cleri_t * k_start = cleri_keyword(CLERI_GID_K_START, "start", CLERI_CASE_SENSITIVE);
+ cleri_t * k_startup_time = cleri_keyword(CLERI_GID_K_STARTUP_TIME, "startup_time", CLERI_CASE_SENSITIVE);
+ cleri_t * k_status = cleri_keyword(CLERI_GID_K_STATUS, "status", CLERI_CASE_SENSITIVE);
+ cleri_t * k_stddev = cleri_keyword(CLERI_GID_K_STDDEV, "stddev", CLERI_CASE_SENSITIVE);
+ cleri_t * k_string = cleri_keyword(CLERI_GID_K_STRING, "string", CLERI_CASE_SENSITIVE);
+ cleri_t * k_suffix = cleri_keyword(CLERI_GID_K_SUFFIX, "suffix", CLERI_CASE_SENSITIVE);
+ cleri_t * k_sum = cleri_keyword(CLERI_GID_K_SUM, "sum", CLERI_CASE_SENSITIVE);
+ cleri_t * k_symmetric_difference = cleri_choice(
+ CLERI_GID_K_SYMMETRIC_DIFFERENCE,
+ CLERI_FIRST_MATCH,
+ 2,
+ cleri_token(CLERI_NONE, "^"),
+ cleri_keyword(CLERI_NONE, "symmetric_difference", CLERI_CASE_SENSITIVE)
+ );
+ cleri_t * k_sync_progress = cleri_keyword(CLERI_GID_K_SYNC_PROGRESS, "sync_progress", CLERI_CASE_SENSITIVE);
+ cleri_t * k_tag = cleri_keyword(CLERI_GID_K_TAG, "tag", CLERI_CASE_SENSITIVE);
+ cleri_t * k_tags = cleri_keyword(CLERI_GID_K_TAGS, "tags", CLERI_CASE_SENSITIVE);
+ cleri_t * k_tee_pipe_name = cleri_keyword(CLERI_GID_K_TEE_PIPE_NAME, "tee_pipe_name", CLERI_CASE_SENSITIVE);
+ cleri_t * k_time_precision = cleri_keyword(CLERI_GID_K_TIME_PRECISION, "time_precision", CLERI_CASE_SENSITIVE);
+ cleri_t * k_timeit = cleri_keyword(CLERI_GID_K_TIMEIT, "timeit", CLERI_CASE_SENSITIVE);
+ cleri_t * k_timeval = cleri_keyword(CLERI_GID_K_TIMEVAL, "timeval", CLERI_CASE_SENSITIVE);
+ cleri_t * k_timezone = cleri_keyword(CLERI_GID_K_TIMEZONE, "timezone", CLERI_CASE_SENSITIVE);
+ cleri_t * k_to = cleri_keyword(CLERI_GID_K_TO, "to", CLERI_CASE_SENSITIVE);
+ cleri_t * k_true = cleri_keyword(CLERI_GID_K_TRUE, "true", CLERI_CASE_SENSITIVE);
+ cleri_t * k_type = cleri_keyword(CLERI_GID_K_TYPE, "type", CLERI_CASE_SENSITIVE);
+ cleri_t * k_union = cleri_choice(
+ CLERI_GID_K_UNION,
+ CLERI_FIRST_MATCH,
+ 2,
+ cleri_tokens(CLERI_NONE, ", |"),
+ cleri_keyword(CLERI_NONE, "union", CLERI_CASE_SENSITIVE)
+ );
+ cleri_t * k_untag = cleri_keyword(CLERI_GID_K_UNTAG, "untag", CLERI_CASE_SENSITIVE);
+ cleri_t * k_uptime = cleri_keyword(CLERI_GID_K_UPTIME, "uptime", CLERI_CASE_SENSITIVE);
+ cleri_t * k_user = cleri_keyword(CLERI_GID_K_USER, "user", CLERI_CASE_SENSITIVE);
+ cleri_t * k_users = cleri_keyword(CLERI_GID_K_USERS, "users", CLERI_CASE_SENSITIVE);
+ cleri_t * k_using = cleri_keyword(CLERI_GID_K_USING, "using", CLERI_CASE_SENSITIVE);
+ cleri_t * k_uuid = cleri_keyword(CLERI_GID_K_UUID, "uuid", CLERI_CASE_SENSITIVE);
+ cleri_t * k_variance = cleri_keyword(CLERI_GID_K_VARIANCE, "variance", CLERI_CASE_SENSITIVE);
+ cleri_t * k_version = cleri_keyword(CLERI_GID_K_VERSION, "version", CLERI_CASE_SENSITIVE);
+ cleri_t * k_warning = cleri_keyword(CLERI_GID_K_WARNING, "warning", CLERI_CASE_SENSITIVE);
+ cleri_t * k_where = cleri_keyword(CLERI_GID_K_WHERE, "where", CLERI_CASE_SENSITIVE);
+ cleri_t * k_who_am_i = cleri_keyword(CLERI_GID_K_WHO_AM_I, "who_am_i", CLERI_CASE_SENSITIVE);
+ cleri_t * k_write = cleri_keyword(CLERI_GID_K_WRITE, "write", CLERI_CASE_SENSITIVE);
+ cleri_t * c_difference = cleri_choice(
+ CLERI_GID_C_DIFFERENCE,
+ CLERI_FIRST_MATCH,
+ 2,
+ cleri_token(CLERI_NONE, "-"),
+ k_difference
+ );
+ cleri_t * access_keywords = cleri_choice(
+ CLERI_GID_ACCESS_KEYWORDS,
+ CLERI_FIRST_MATCH,
+ 14,
+ k_read,
+ k_write,
+ k_modify,
+ k_full,
+ k_select,
+ k_show,
+ k_list,
+ k_count,
+ k_create,
+ k_insert,
+ k_drop,
+ k_grant,
+ k_revoke,
+ k_alter
+ );
+ cleri_t * _boolean = cleri_choice(
+ CLERI_GID__BOOLEAN,
+ CLERI_FIRST_MATCH,
+ 2,
+ k_true,
+ k_false
+ );
+ cleri_t * log_keywords = cleri_choice(
+ CLERI_GID_LOG_KEYWORDS,
+ CLERI_FIRST_MATCH,
+ 5,
+ k_debug,
+ k_info,
+ k_warning,
+ k_error,
+ k_critical
+ );
+ cleri_t * int_expr = cleri_prio(
+ CLERI_GID_INT_EXPR,
+ 3,
+ r_integer,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ cleri_tokens(CLERI_NONE, "+ - * % /"),
+ CLERI_THIS
+ )
+ );
+ cleri_t * string = cleri_choice(
+ CLERI_GID_STRING,
+ CLERI_FIRST_MATCH,
+ 2,
+ r_singleq_str,
+ r_doubleq_str
+ );
+ cleri_t * time_expr = cleri_prio(
+ CLERI_GID_TIME_EXPR,
+ 6,
+ r_time_str,
+ k_now,
+ string,
+ r_integer,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ cleri_tokens(CLERI_NONE, "+ - * % /"),
+ CLERI_THIS
+ )
+ );
+ cleri_t * series_columns = cleri_list(CLERI_GID_SERIES_COLUMNS, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 7,
+ k_name,
+ k_type,
+ k_length,
+ k_start,
+ k_end,
+ k_shard_duration,
+ k_pool
+ ), cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * shard_columns = cleri_list(CLERI_GID_SHARD_COLUMNS, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 8,
+ k_sid,
+ k_pool,
+ k_server,
+ k_size,
+ k_start,
+ k_end,
+ k_type,
+ k_status
+ ), cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * server_columns = cleri_list(CLERI_GID_SERVER_COLUMNS, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 29,
+ k_address,
+ k_buffer_path,
+ k_buffer_size,
+ k_dbpath,
+ k_ip_support,
+ k_libuv,
+ k_name,
+ k_port,
+ k_uuid,
+ k_pool,
+ k_version,
+ k_online,
+ k_startup_time,
+ k_status,
+ k_active_handles,
+ k_active_tasks,
+ k_fifo_files,
+ k_idle_percentage,
+ k_idle_time,
+ k_log_level,
+ k_max_open_files,
+ k_mem_usage,
+ k_open_files,
+ k_received_points,
+ k_reindex_progress,
+ k_selected_points,
+ k_sync_progress,
+ k_tee_pipe_name,
+ k_uptime
+ ), cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * group_columns = cleri_list(CLERI_GID_GROUP_COLUMNS, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 3,
+ k_expression,
+ k_name,
+ k_series
+ ), cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * user_columns = cleri_list(CLERI_GID_USER_COLUMNS, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ k_name,
+ k_access
+ ), cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * tag_columns = cleri_list(CLERI_GID_TAG_COLUMNS, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ k_name,
+ k_series
+ ), cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * pool_props = cleri_choice(
+ CLERI_GID_POOL_PROPS,
+ CLERI_FIRST_MATCH,
+ 3,
+ k_pool,
+ k_servers,
+ k_series
+ );
+ cleri_t * pool_columns = cleri_list(CLERI_GID_POOL_COLUMNS, pool_props, cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * bool_operator = cleri_tokens(CLERI_GID_BOOL_OPERATOR, "== !=");
+ cleri_t * int_operator = cleri_tokens(CLERI_GID_INT_OPERATOR, "== != <= >= < >");
+ cleri_t * str_operator = cleri_tokens(CLERI_GID_STR_OPERATOR, "== != <= >= !~ < > ~");
+ cleri_t * where_group = cleri_sequence(
+ CLERI_GID_WHERE_GROUP,
+ 2,
+ k_where,
+ cleri_prio(
+ CLERI_NONE,
+ 5,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_series,
+ int_operator,
+ int_expr
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ k_expression,
+ k_name
+ ),
+ str_operator,
+ string
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_and,
+ CLERI_THIS
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_or,
+ CLERI_THIS
+ )
+ )
+ );
+ cleri_t * where_tag = cleri_sequence(
+ CLERI_GID_WHERE_TAG,
+ 2,
+ k_where,
+ cleri_prio(
+ CLERI_NONE,
+ 5,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_name,
+ str_operator,
+ string
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_series,
+ int_operator,
+ int_expr
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_and,
+ CLERI_THIS
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_or,
+ CLERI_THIS
+ )
+ )
+ );
+ cleri_t * where_pool = cleri_sequence(
+ CLERI_GID_WHERE_POOL,
+ 2,
+ k_where,
+ cleri_prio(
+ CLERI_NONE,
+ 4,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ pool_props,
+ int_operator,
+ int_expr
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_and,
+ CLERI_THIS
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_or,
+ CLERI_THIS
+ )
+ )
+ );
+ cleri_t * where_series = cleri_sequence(
+ CLERI_GID_WHERE_SERIES,
+ 2,
+ k_where,
+ cleri_prio(
+ CLERI_NONE,
+ 7,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ k_length,
+ k_pool
+ ),
+ int_operator,
+ int_expr
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_name,
+ str_operator,
+ string
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 3,
+ k_start,
+ k_end,
+ k_shard_duration
+ ),
+ int_operator,
+ time_expr
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_type,
+ bool_operator,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 3,
+ k_string,
+ k_integer,
+ k_float
+ )
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_and,
+ CLERI_THIS
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_or,
+ CLERI_THIS
+ )
+ )
+ );
+ cleri_t * where_server = cleri_sequence(
+ CLERI_GID_WHERE_SERVER,
+ 2,
+ k_where,
+ cleri_prio(
+ CLERI_NONE,
+ 7,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 15,
+ k_active_handles,
+ k_active_tasks,
+ k_buffer_size,
+ k_fifo_files,
+ k_idle_percentage,
+ k_idle_time,
+ k_port,
+ k_pool,
+ k_startup_time,
+ k_max_open_files,
+ k_mem_usage,
+ k_open_files,
+ k_received_points,
+ k_selected_points,
+ k_uptime
+ ),
+ int_operator,
+ int_expr
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 12,
+ k_address,
+ k_buffer_path,
+ k_dbpath,
+ k_ip_support,
+ k_libuv,
+ k_name,
+ k_uuid,
+ k_version,
+ k_status,
+ k_reindex_progress,
+ k_sync_progress,
+ k_tee_pipe_name
+ ),
+ str_operator,
+ string
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_online,
+ bool_operator,
+ _boolean
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_log_level,
+ int_operator,
+ log_keywords
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_and,
+ CLERI_THIS
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_or,
+ CLERI_THIS
+ )
+ )
+ );
+ cleri_t * where_shard = cleri_sequence(
+ CLERI_GID_WHERE_SHARD,
+ 2,
+ k_where,
+ cleri_prio(
+ CLERI_NONE,
+ 7,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 3,
+ k_sid,
+ k_pool,
+ k_size
+ ),
+ int_operator,
+ int_expr
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 2,
+ k_server,
+ k_status
+ ),
+ str_operator,
+ string
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ k_start,
+ k_end
+ ),
+ int_operator,
+ time_expr
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_type,
+ bool_operator,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ k_number,
+ k_log
+ )
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_and,
+ CLERI_THIS
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_or,
+ CLERI_THIS
+ )
+ )
+ );
+ cleri_t * where_user = cleri_sequence(
+ CLERI_GID_WHERE_USER,
+ 2,
+ k_where,
+ cleri_prio(
+ CLERI_NONE,
+ 5,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_name,
+ str_operator,
+ string
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ k_access,
+ int_operator,
+ access_keywords
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_and,
+ CLERI_THIS
+ ),
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ k_or,
+ CLERI_THIS
+ )
+ )
+ );
+ cleri_t * series_setopr = cleri_choice(
+ CLERI_GID_SERIES_SETOPR,
+ CLERI_FIRST_MATCH,
+ 4,
+ k_union,
+ c_difference,
+ k_intersection,
+ k_symmetric_difference
+ );
+ cleri_t * series_parentheses = cleri_sequence(
+ CLERI_GID_SERIES_PARENTHESES,
+ 3,
+ cleri_token(CLERI_NONE, "("),
+ CLERI_THIS,
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * series_all = cleri_choice(
+ CLERI_GID_SERIES_ALL,
+ CLERI_FIRST_MATCH,
+ 2,
+ cleri_token(CLERI_NONE, "*"),
+ k_all
+ );
+ cleri_t * series_name = cleri_dup(CLERI_GID_SERIES_NAME, string);
+ cleri_t * group_name = cleri_dup(CLERI_GID_GROUP_NAME, r_grave_str);
+ cleri_t * tag_name = cleri_dup(CLERI_GID_TAG_NAME, r_grave_str);
+ cleri_t * series_re = cleri_dup(CLERI_GID_SERIES_RE, r_regex);
+ cleri_t * uuid = cleri_choice(
+ CLERI_GID_UUID,
+ CLERI_FIRST_MATCH,
+ 2,
+ r_uuid_str,
+ string
+ );
+ cleri_t * group_tag_match = cleri_dup(CLERI_GID_GROUP_TAG_MATCH, r_grave_str);
+ cleri_t * series_match = cleri_prio(
+ CLERI_GID_SERIES_MATCH,
+ 4,
+ cleri_list(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 4,
+ series_all,
+ series_name,
+ group_tag_match,
+ series_re
+ ), series_setopr, 1, 0, 0),
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 4,
+ series_all,
+ series_name,
+ group_tag_match,
+ series_re
+ ),
+ series_parentheses,
+ cleri_sequence(
+ CLERI_NONE,
+ 3,
+ CLERI_THIS,
+ series_setopr,
+ CLERI_THIS
+ )
+ );
+ cleri_t * limit_expr = cleri_sequence(
+ CLERI_GID_LIMIT_EXPR,
+ 2,
+ k_limit,
+ int_expr
+ );
+ cleri_t * before_expr = cleri_sequence(
+ CLERI_GID_BEFORE_EXPR,
+ 2,
+ k_before,
+ time_expr
+ );
+ cleri_t * after_expr = cleri_sequence(
+ CLERI_GID_AFTER_EXPR,
+ 2,
+ k_after,
+ time_expr
+ );
+ cleri_t * between_expr = cleri_sequence(
+ CLERI_GID_BETWEEN_EXPR,
+ 4,
+ k_between,
+ time_expr,
+ k_and,
+ time_expr
+ );
+ cleri_t * access_expr = cleri_list(CLERI_GID_ACCESS_EXPR, access_keywords, cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * prefix_expr = cleri_sequence(
+ CLERI_GID_PREFIX_EXPR,
+ 2,
+ k_prefix,
+ string
+ );
+ cleri_t * suffix_expr = cleri_sequence(
+ CLERI_GID_SUFFIX_EXPR,
+ 2,
+ k_suffix,
+ string
+ );
+ cleri_t * f_all = cleri_choice(
+ CLERI_GID_F_ALL,
+ CLERI_FIRST_MATCH,
+ 2,
+ cleri_token(CLERI_NONE, "*"),
+ k_all
+ );
+ cleri_t * f_points = cleri_dup(CLERI_GID_F_POINTS, k_points);
+ cleri_t * f_difference = cleri_sequence(
+ CLERI_GID_F_DIFFERENCE,
+ 4,
+ k_difference,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_derivative = cleri_sequence(
+ CLERI_GID_F_DERIVATIVE,
+ 4,
+ k_derivative,
+ cleri_token(CLERI_NONE, "("),
+ cleri_list(CLERI_NONE, time_expr, cleri_token(CLERI_NONE, ","), 0, 2, 0),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_mean = cleri_sequence(
+ CLERI_GID_F_MEAN,
+ 4,
+ k_mean,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_median = cleri_sequence(
+ CLERI_GID_F_MEDIAN,
+ 4,
+ k_median,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_median_low = cleri_sequence(
+ CLERI_GID_F_MEDIAN_LOW,
+ 4,
+ k_median_low,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_median_high = cleri_sequence(
+ CLERI_GID_F_MEDIAN_HIGH,
+ 4,
+ k_median_high,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_sum = cleri_sequence(
+ CLERI_GID_F_SUM,
+ 4,
+ k_sum,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_min = cleri_sequence(
+ CLERI_GID_F_MIN,
+ 4,
+ k_min,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_max = cleri_sequence(
+ CLERI_GID_F_MAX,
+ 4,
+ k_max,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_count = cleri_sequence(
+ CLERI_GID_F_COUNT,
+ 4,
+ k_count,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_variance = cleri_sequence(
+ CLERI_GID_F_VARIANCE,
+ 4,
+ k_variance,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_pvariance = cleri_sequence(
+ CLERI_GID_F_PVARIANCE,
+ 4,
+ k_pvariance,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_stddev = cleri_sequence(
+ CLERI_GID_F_STDDEV,
+ 4,
+ k_stddev,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_first = cleri_sequence(
+ CLERI_GID_F_FIRST,
+ 4,
+ k_first,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_last = cleri_sequence(
+ CLERI_GID_F_LAST,
+ 4,
+ k_last,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, time_expr),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_timeval = cleri_sequence(
+ CLERI_GID_F_TIMEVAL,
+ 3,
+ k_timeval,
+ cleri_token(CLERI_NONE, "("),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_interval = cleri_sequence(
+ CLERI_GID_F_INTERVAL,
+ 3,
+ k_interval,
+ cleri_token(CLERI_NONE, "("),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_filter = cleri_sequence(
+ CLERI_GID_F_FILTER,
+ 5,
+ k_filter,
+ cleri_token(CLERI_NONE, "("),
+ cleri_optional(CLERI_NONE, str_operator),
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 7,
+ string,
+ r_integer,
+ r_float,
+ r_regex,
+ k_nan,
+ k_inf,
+ k_ninf
+ ),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * f_limit = cleri_sequence(
+ CLERI_GID_F_LIMIT,
+ 6,
+ k_limit,
+ cleri_token(CLERI_NONE, "("),
+ int_expr,
+ cleri_token(CLERI_NONE, ","),
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 13,
+ k_mean,
+ k_median,
+ k_median_high,
+ k_median_low,
+ k_sum,
+ k_min,
+ k_max,
+ k_count,
+ k_variance,
+ k_pvariance,
+ k_stddev,
+ k_first,
+ k_last
+ ),
+ cleri_token(CLERI_NONE, ")")
+ );
+ cleri_t * aggregate_functions = cleri_list(CLERI_GID_AGGREGATE_FUNCTIONS, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 21,
+ f_all,
+ f_limit,
+ f_mean,
+ f_sum,
+ f_median,
+ f_median_low,
+ f_median_high,
+ f_min,
+ f_max,
+ f_count,
+ f_variance,
+ f_pvariance,
+ f_stddev,
+ f_first,
+ f_last,
+ f_timeval,
+ f_interval,
+ f_difference,
+ f_derivative,
+ f_filter,
+ f_points
+ ), cleri_token(CLERI_NONE, "=>"), 1, 0, 0);
+ cleri_t * select_aggregate = cleri_sequence(
+ CLERI_GID_SELECT_AGGREGATE,
+ 3,
+ aggregate_functions,
+ cleri_optional(CLERI_NONE, prefix_expr),
+ cleri_optional(CLERI_NONE, suffix_expr)
+ );
+ cleri_t * select_aggregates = cleri_list(CLERI_GID_SELECT_AGGREGATES, select_aggregate, cleri_token(CLERI_NONE, ","), 1, 0, 0);
+ cleri_t * merge_as = cleri_sequence(
+ CLERI_GID_MERGE_AS,
+ 4,
+ k_merge,
+ k_as,
+ string,
+ cleri_optional(CLERI_NONE, cleri_sequence(
+ CLERI_NONE,
+ 2,
+ k_using,
+ aggregate_functions
+ ))
+ );
+ cleri_t * set_address = cleri_sequence(
+ CLERI_GID_SET_ADDRESS,
+ 3,
+ k_set,
+ k_address,
+ string
+ );
+ cleri_t * set_tee_pipe_name = cleri_sequence(
+ CLERI_GID_SET_TEE_PIPE_NAME,
+ 3,
+ k_set,
+ k_tee_pipe_name,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ k_false,
+ string
+ )
+ );
+ cleri_t * set_backup_mode = cleri_sequence(
+ CLERI_GID_SET_BACKUP_MODE,
+ 3,
+ k_set,
+ k_backup_mode,
+ _boolean
+ );
+ cleri_t * set_drop_threshold = cleri_sequence(
+ CLERI_GID_SET_DROP_THRESHOLD,
+ 3,
+ k_set,
+ k_drop_threshold,
+ r_float
+ );
+ cleri_t * set_expression = cleri_sequence(
+ CLERI_GID_SET_EXPRESSION,
+ 3,
+ k_set,
+ k_expression,
+ r_regex
+ );
+ cleri_t * set_ignore_threshold = cleri_sequence(
+ CLERI_GID_SET_IGNORE_THRESHOLD,
+ 3,
+ k_set,
+ k_ignore_threshold,
+ _boolean
+ );
+ cleri_t * set_list_limit = cleri_sequence(
+ CLERI_GID_SET_LIST_LIMIT,
+ 3,
+ k_set,
+ k_list_limit,
+ r_uinteger
+ );
+ cleri_t * set_log_level = cleri_sequence(
+ CLERI_GID_SET_LOG_LEVEL,
+ 3,
+ k_set,
+ k_log_level,
+ log_keywords
+ );
+ cleri_t * set_name = cleri_sequence(
+ CLERI_GID_SET_NAME,
+ 3,
+ k_set,
+ k_name,
+ string
+ );
+ cleri_t * set_password = cleri_sequence(
+ CLERI_GID_SET_PASSWORD,
+ 3,
+ k_set,
+ k_password,
+ string
+ );
+ cleri_t * set_port = cleri_sequence(
+ CLERI_GID_SET_PORT,
+ 3,
+ k_set,
+ k_port,
+ r_uinteger
+ );
+ cleri_t * set_select_points_limit = cleri_sequence(
+ CLERI_GID_SET_SELECT_POINTS_LIMIT,
+ 3,
+ k_set,
+ k_select_points_limit,
+ r_uinteger
+ );
+ cleri_t * set_timezone = cleri_sequence(
+ CLERI_GID_SET_TIMEZONE,
+ 3,
+ k_set,
+ k_timezone,
+ string
+ );
+ cleri_t * tag_series = cleri_sequence(
+ CLERI_GID_TAG_SERIES,
+ 2,
+ k_tag,
+ tag_name
+ );
+ cleri_t * untag_series = cleri_sequence(
+ CLERI_GID_UNTAG_SERIES,
+ 2,
+ k_untag,
+ tag_name
+ );
+ cleri_t * set_expiration_num = cleri_sequence(
+ CLERI_GID_SET_EXPIRATION_NUM,
+ 4,
+ k_set,
+ k_expiration_num,
+ time_expr,
+ cleri_optional(CLERI_NONE, set_ignore_threshold)
+ );
+ cleri_t * set_expiration_log = cleri_sequence(
+ CLERI_GID_SET_EXPIRATION_LOG,
+ 4,
+ k_set,
+ k_expiration_log,
+ time_expr,
+ cleri_optional(CLERI_NONE, set_ignore_threshold)
+ );
+ cleri_t * alter_database = cleri_sequence(
+ CLERI_GID_ALTER_DATABASE,
+ 2,
+ k_database,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 6,
+ set_drop_threshold,
+ set_list_limit,
+ set_select_points_limit,
+ set_timezone,
+ set_expiration_num,
+ set_expiration_log
+ )
+ );
+ cleri_t * alter_group = cleri_sequence(
+ CLERI_GID_ALTER_GROUP,
+ 3,
+ k_group,
+ group_name,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ set_expression,
+ set_name
+ )
+ );
+ cleri_t * alter_tag = cleri_sequence(
+ CLERI_GID_ALTER_TAG,
+ 3,
+ k_tag,
+ tag_name,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 1,
+ set_name
+ )
+ );
+ cleri_t * alter_server = cleri_sequence(
+ CLERI_GID_ALTER_SERVER,
+ 3,
+ k_server,
+ uuid,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 5,
+ set_log_level,
+ set_backup_mode,
+ set_tee_pipe_name,
+ set_address,
+ set_port
+ )
+ );
+ cleri_t * alter_servers = cleri_sequence(
+ CLERI_GID_ALTER_SERVERS,
+ 3,
+ k_servers,
+ cleri_optional(CLERI_NONE, where_server),
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ set_log_level,
+ set_tee_pipe_name
+ )
+ );
+ cleri_t * alter_user = cleri_sequence(
+ CLERI_GID_ALTER_USER,
+ 3,
+ k_user,
+ string,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ set_password,
+ set_name
+ )
+ );
+ cleri_t * alter_series = cleri_sequence(
+ CLERI_GID_ALTER_SERIES,
+ 4,
+ k_series,
+ series_match,
+ cleri_optional(CLERI_NONE, where_series),
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 2,
+ tag_series,
+ untag_series
+ )
+ );
+ cleri_t * count_groups = cleri_sequence(
+ CLERI_GID_COUNT_GROUPS,
+ 2,
+ k_groups,
+ cleri_optional(CLERI_NONE, where_group)
+ );
+ cleri_t * count_tags = cleri_sequence(
+ CLERI_GID_COUNT_TAGS,
+ 2,
+ k_tags,
+ cleri_optional(CLERI_NONE, where_tag)
+ );
+ cleri_t * count_pools = cleri_sequence(
+ CLERI_GID_COUNT_POOLS,
+ 2,
+ k_pools,
+ cleri_optional(CLERI_NONE, where_pool)
+ );
+ cleri_t * count_series = cleri_sequence(
+ CLERI_GID_COUNT_SERIES,
+ 3,
+ k_series,
+ cleri_optional(CLERI_NONE, series_match),
+ cleri_optional(CLERI_NONE, where_series)
+ );
+ cleri_t * count_servers = cleri_sequence(
+ CLERI_GID_COUNT_SERVERS,
+ 2,
+ k_servers,
+ cleri_optional(CLERI_NONE, where_server)
+ );
+ cleri_t * count_servers_received = cleri_sequence(
+ CLERI_GID_COUNT_SERVERS_RECEIVED,
+ 3,
+ k_servers,
+ k_received_points,
+ cleri_optional(CLERI_NONE, where_server)
+ );
+ cleri_t * count_servers_selected = cleri_sequence(
+ CLERI_GID_COUNT_SERVERS_SELECTED,
+ 3,
+ k_servers,
+ k_selected_points,
+ cleri_optional(CLERI_NONE, where_server)
+ );
+ cleri_t * count_shards = cleri_sequence(
+ CLERI_GID_COUNT_SHARDS,
+ 2,
+ k_shards,
+ cleri_optional(CLERI_NONE, where_shard)
+ );
+ cleri_t * count_shards_size = cleri_sequence(
+ CLERI_GID_COUNT_SHARDS_SIZE,
+ 3,
+ k_shards,
+ k_size,
+ cleri_optional(CLERI_NONE, where_shard)
+ );
+ cleri_t * count_users = cleri_sequence(
+ CLERI_GID_COUNT_USERS,
+ 2,
+ k_users,
+ cleri_optional(CLERI_NONE, where_user)
+ );
+ cleri_t * count_series_length = cleri_sequence(
+ CLERI_GID_COUNT_SERIES_LENGTH,
+ 4,
+ k_series,
+ k_length,
+ cleri_optional(CLERI_NONE, series_match),
+ cleri_optional(CLERI_NONE, where_series)
+ );
+ cleri_t * create_group = cleri_sequence(
+ CLERI_GID_CREATE_GROUP,
+ 4,
+ k_group,
+ group_name,
+ k_for,
+ r_regex
+ );
+ cleri_t * create_user = cleri_sequence(
+ CLERI_GID_CREATE_USER,
+ 3,
+ k_user,
+ string,
+ set_password
+ );
+ cleri_t * drop_group = cleri_sequence(
+ CLERI_GID_DROP_GROUP,
+ 2,
+ k_group,
+ group_name
+ );
+ cleri_t * drop_tag = cleri_sequence(
+ CLERI_GID_DROP_TAG,
+ 2,
+ k_tag,
+ tag_name
+ );
+ cleri_t * drop_series = cleri_sequence(
+ CLERI_GID_DROP_SERIES,
+ 4,
+ k_series,
+ cleri_optional(CLERI_NONE, series_match),
+ cleri_optional(CLERI_NONE, where_series),
+ cleri_optional(CLERI_NONE, set_ignore_threshold)
+ );
+ cleri_t * drop_shards = cleri_sequence(
+ CLERI_GID_DROP_SHARDS,
+ 3,
+ k_shards,
+ cleri_optional(CLERI_NONE, where_shard),
+ cleri_optional(CLERI_NONE, set_ignore_threshold)
+ );
+ cleri_t * drop_server = cleri_sequence(
+ CLERI_GID_DROP_SERVER,
+ 2,
+ k_server,
+ uuid
+ );
+ cleri_t * drop_user = cleri_sequence(
+ CLERI_GID_DROP_USER,
+ 2,
+ k_user,
+ string
+ );
+ cleri_t * grant_user = cleri_sequence(
+ CLERI_GID_GRANT_USER,
+ 3,
+ k_user,
+ string,
+ cleri_optional(CLERI_NONE, set_password)
+ );
+ cleri_t * list_groups = cleri_sequence(
+ CLERI_GID_LIST_GROUPS,
+ 3,
+ k_groups,
+ cleri_optional(CLERI_NONE, group_columns),
+ cleri_optional(CLERI_NONE, where_group)
+ );
+ cleri_t * list_tags = cleri_sequence(
+ CLERI_GID_LIST_TAGS,
+ 3,
+ k_tags,
+ cleri_optional(CLERI_NONE, tag_columns),
+ cleri_optional(CLERI_NONE, where_tag)
+ );
+ cleri_t * list_pools = cleri_sequence(
+ CLERI_GID_LIST_POOLS,
+ 3,
+ k_pools,
+ cleri_optional(CLERI_NONE, pool_columns),
+ cleri_optional(CLERI_NONE, where_pool)
+ );
+ cleri_t * list_series = cleri_sequence(
+ CLERI_GID_LIST_SERIES,
+ 4,
+ k_series,
+ cleri_optional(CLERI_NONE, series_columns),
+ cleri_optional(CLERI_NONE, series_match),
+ cleri_optional(CLERI_NONE, where_series)
+ );
+ cleri_t * list_servers = cleri_sequence(
+ CLERI_GID_LIST_SERVERS,
+ 3,
+ k_servers,
+ cleri_optional(CLERI_NONE, server_columns),
+ cleri_optional(CLERI_NONE, where_server)
+ );
+ cleri_t * list_shards = cleri_sequence(
+ CLERI_GID_LIST_SHARDS,
+ 3,
+ k_shards,
+ cleri_optional(CLERI_NONE, shard_columns),
+ cleri_optional(CLERI_NONE, where_shard)
+ );
+ cleri_t * list_users = cleri_sequence(
+ CLERI_GID_LIST_USERS,
+ 3,
+ k_users,
+ cleri_optional(CLERI_NONE, user_columns),
+ cleri_optional(CLERI_NONE, where_user)
+ );
+ cleri_t * revoke_user = cleri_sequence(
+ CLERI_GID_REVOKE_USER,
+ 2,
+ k_user,
+ string
+ );
+ cleri_t * alter_stmt = cleri_sequence(
+ CLERI_GID_ALTER_STMT,
+ 2,
+ k_alter,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 7,
+ alter_series,
+ alter_user,
+ alter_group,
+ alter_tag,
+ alter_server,
+ alter_servers,
+ alter_database
+ )
+ );
+ cleri_t * calc_stmt = cleri_dup(CLERI_GID_CALC_STMT, time_expr);
+ cleri_t * count_stmt = cleri_sequence(
+ CLERI_GID_COUNT_STMT,
+ 2,
+ k_count,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 11,
+ count_groups,
+ count_pools,
+ count_series,
+ count_servers,
+ count_servers_received,
+ count_servers_selected,
+ count_shards,
+ count_shards_size,
+ count_users,
+ count_tags,
+ count_series_length
+ )
+ );
+ cleri_t * create_stmt = cleri_sequence(
+ CLERI_GID_CREATE_STMT,
+ 2,
+ k_create,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 2,
+ create_group,
+ create_user
+ )
+ );
+ cleri_t * drop_stmt = cleri_sequence(
+ CLERI_GID_DROP_STMT,
+ 2,
+ k_drop,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 6,
+ drop_group,
+ drop_tag,
+ drop_series,
+ drop_shards,
+ drop_server,
+ drop_user
+ )
+ );
+ cleri_t * grant_stmt = cleri_sequence(
+ CLERI_GID_GRANT_STMT,
+ 4,
+ k_grant,
+ access_expr,
+ k_to,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 1,
+ grant_user
+ )
+ );
+ cleri_t * list_stmt = cleri_sequence(
+ CLERI_GID_LIST_STMT,
+ 3,
+ k_list,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 7,
+ list_series,
+ list_tags,
+ list_users,
+ list_shards,
+ list_groups,
+ list_servers,
+ list_pools
+ ),
+ cleri_optional(CLERI_NONE, limit_expr)
+ );
+ cleri_t * revoke_stmt = cleri_sequence(
+ CLERI_GID_REVOKE_STMT,
+ 4,
+ k_revoke,
+ access_expr,
+ k_from,
+ cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 1,
+ revoke_user
+ )
+ );
+ cleri_t * select_stmt = cleri_sequence(
+ CLERI_GID_SELECT_STMT,
+ 7,
+ k_select,
+ select_aggregates,
+ k_from,
+ series_match,
+ cleri_optional(CLERI_NONE, where_series),
+ cleri_optional(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 3,
+ after_expr,
+ between_expr,
+ before_expr
+ )),
+ cleri_optional(CLERI_NONE, merge_as)
+ );
+ cleri_t * show_stmt = cleri_sequence(
+ CLERI_GID_SHOW_STMT,
+ 2,
+ k_show,
+ cleri_list(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 37,
+ k_active_handles,
+ k_active_tasks,
+ k_buffer_path,
+ k_buffer_size,
+ k_dbname,
+ k_dbpath,
+ k_drop_threshold,
+ k_duration_log,
+ k_duration_num,
+ k_fifo_files,
+ k_expiration_log,
+ k_expiration_num,
+ k_idle_percentage,
+ k_idle_time,
+ k_ip_support,
+ k_libuv,
+ k_list_limit,
+ k_log_level,
+ k_max_open_files,
+ k_mem_usage,
+ k_open_files,
+ k_pool,
+ k_received_points,
+ k_reindex_progress,
+ k_selected_points,
+ k_select_points_limit,
+ k_server,
+ k_startup_time,
+ k_status,
+ k_sync_progress,
+ k_tee_pipe_name,
+ k_time_precision,
+ k_timezone,
+ k_uptime,
+ k_uuid,
+ k_version,
+ k_who_am_i
+ ), cleri_token(CLERI_NONE, ","), 0, 0, 0)
+ );
+ cleri_t * timeit_stmt = cleri_dup(CLERI_GID_TIMEIT_STMT, k_timeit);
+ cleri_t * help_stmt = cleri_ref();
+ cleri_t * START = cleri_sequence(
+ CLERI_GID_START,
+ 3,
+ cleri_optional(CLERI_NONE, timeit_stmt),
+ cleri_optional(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_FIRST_MATCH,
+ 11,
+ select_stmt,
+ list_stmt,
+ count_stmt,
+ alter_stmt,
+ create_stmt,
+ drop_stmt,
+ grant_stmt,
+ revoke_stmt,
+ show_stmt,
+ calc_stmt,
+ help_stmt
+ )),
+ cleri_optional(CLERI_NONE, r_comment)
+ );
+ cleri_t * help_access = cleri_keyword(CLERI_GID_HELP_ACCESS, "access", CLERI_CASE_SENSITIVE);
+ cleri_t * help_alter_database = cleri_keyword(CLERI_GID_HELP_ALTER_DATABASE, "database", CLERI_CASE_SENSITIVE);
+ cleri_t * help_alter_group = cleri_keyword(CLERI_GID_HELP_ALTER_GROUP, "group", CLERI_CASE_SENSITIVE);
+ cleri_t * help_alter_server = cleri_keyword(CLERI_GID_HELP_ALTER_SERVER, "server", CLERI_CASE_SENSITIVE);
+ cleri_t * help_alter_servers = cleri_keyword(CLERI_GID_HELP_ALTER_SERVERS, "servers", CLERI_CASE_SENSITIVE);
+ cleri_t * help_alter_user = cleri_keyword(CLERI_GID_HELP_ALTER_USER, "user", CLERI_CASE_SENSITIVE);
+ cleri_t * help_alter = cleri_sequence(
+ CLERI_GID_HELP_ALTER,
+ 2,
+ k_alter,
+ cleri_optional(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 5,
+ help_alter_database,
+ help_alter_group,
+ help_alter_server,
+ help_alter_servers,
+ help_alter_user
+ ))
+ );
+ cleri_t * help_count_groups = cleri_keyword(CLERI_GID_HELP_COUNT_GROUPS, "groups", CLERI_CASE_SENSITIVE);
+ cleri_t * help_count_pools = cleri_keyword(CLERI_GID_HELP_COUNT_POOLS, "pools", CLERI_CASE_SENSITIVE);
+ cleri_t * help_count_series = cleri_keyword(CLERI_GID_HELP_COUNT_SERIES, "series", CLERI_CASE_SENSITIVE);
+ cleri_t * help_count_servers = cleri_keyword(CLERI_GID_HELP_COUNT_SERVERS, "servers", CLERI_CASE_SENSITIVE);
+ cleri_t * help_count_shards = cleri_keyword(CLERI_GID_HELP_COUNT_SHARDS, "shards", CLERI_CASE_SENSITIVE);
+ cleri_t * help_count_users = cleri_keyword(CLERI_GID_HELP_COUNT_USERS, "users", CLERI_CASE_SENSITIVE);
+ cleri_t * help_count = cleri_sequence(
+ CLERI_GID_HELP_COUNT,
+ 2,
+ k_count,
+ cleri_optional(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 6,
+ help_count_groups,
+ help_count_pools,
+ help_count_series,
+ help_count_servers,
+ help_count_shards,
+ help_count_users
+ ))
+ );
+ cleri_t * help_create_group = cleri_keyword(CLERI_GID_HELP_CREATE_GROUP, "group", CLERI_CASE_SENSITIVE);
+ cleri_t * help_create_user = cleri_keyword(CLERI_GID_HELP_CREATE_USER, "user", CLERI_CASE_SENSITIVE);
+ cleri_t * help_create = cleri_sequence(
+ CLERI_GID_HELP_CREATE,
+ 2,
+ k_create,
+ cleri_optional(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 2,
+ help_create_group,
+ help_create_user
+ ))
+ );
+ cleri_t * help_drop_group = cleri_keyword(CLERI_GID_HELP_DROP_GROUP, "group", CLERI_CASE_SENSITIVE);
+ cleri_t * help_drop_series = cleri_keyword(CLERI_GID_HELP_DROP_SERIES, "series", CLERI_CASE_SENSITIVE);
+ cleri_t * help_drop_server = cleri_keyword(CLERI_GID_HELP_DROP_SERVER, "server", CLERI_CASE_SENSITIVE);
+ cleri_t * help_drop_shards = cleri_keyword(CLERI_GID_HELP_DROP_SHARDS, "shards", CLERI_CASE_SENSITIVE);
+ cleri_t * help_drop_user = cleri_keyword(CLERI_GID_HELP_DROP_USER, "user", CLERI_CASE_SENSITIVE);
+ cleri_t * help_drop = cleri_sequence(
+ CLERI_GID_HELP_DROP,
+ 2,
+ k_drop,
+ cleri_optional(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 5,
+ help_drop_group,
+ help_drop_series,
+ help_drop_server,
+ help_drop_shards,
+ help_drop_user
+ ))
+ );
+ cleri_t * help_functions = cleri_keyword(CLERI_GID_HELP_FUNCTIONS, "functions", CLERI_CASE_SENSITIVE);
+ cleri_t * help_grant = cleri_keyword(CLERI_GID_HELP_GRANT, "grant", CLERI_CASE_SENSITIVE);
+ cleri_t * help_list_groups = cleri_keyword(CLERI_GID_HELP_LIST_GROUPS, "groups", CLERI_CASE_SENSITIVE);
+ cleri_t * help_list_pools = cleri_keyword(CLERI_GID_HELP_LIST_POOLS, "pools", CLERI_CASE_SENSITIVE);
+ cleri_t * help_list_series = cleri_keyword(CLERI_GID_HELP_LIST_SERIES, "series", CLERI_CASE_SENSITIVE);
+ cleri_t * help_list_servers = cleri_keyword(CLERI_GID_HELP_LIST_SERVERS, "servers", CLERI_CASE_SENSITIVE);
+ cleri_t * help_list_shards = cleri_keyword(CLERI_GID_HELP_LIST_SHARDS, "shards", CLERI_CASE_SENSITIVE);
+ cleri_t * help_list_users = cleri_keyword(CLERI_GID_HELP_LIST_USERS, "users", CLERI_CASE_SENSITIVE);
+ cleri_t * help_list = cleri_sequence(
+ CLERI_GID_HELP_LIST,
+ 2,
+ k_list,
+ cleri_optional(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 6,
+ help_list_groups,
+ help_list_pools,
+ help_list_series,
+ help_list_servers,
+ help_list_shards,
+ help_list_users
+ ))
+ );
+ cleri_t * help_noaccess = cleri_keyword(CLERI_GID_HELP_NOACCESS, "noaccess", CLERI_CASE_SENSITIVE);
+ cleri_t * help_revoke = cleri_keyword(CLERI_GID_HELP_REVOKE, "revoke", CLERI_CASE_SENSITIVE);
+ cleri_t * help_select = cleri_keyword(CLERI_GID_HELP_SELECT, "select", CLERI_CASE_SENSITIVE);
+ cleri_t * help_show = cleri_keyword(CLERI_GID_HELP_SHOW, "show", CLERI_CASE_SENSITIVE);
+ cleri_t * help_timeit = cleri_keyword(CLERI_GID_HELP_TIMEIT, "timeit", CLERI_CASE_SENSITIVE);
+ cleri_t * help_timezones = cleri_keyword(CLERI_GID_HELP_TIMEZONES, "timezones", CLERI_CASE_SENSITIVE);
+ cleri_ref_set(help_stmt, cleri_sequence(
+ CLERI_GID_HELP_STMT,
+ 2,
+ k_help,
+ cleri_optional(CLERI_NONE, cleri_choice(
+ CLERI_NONE,
+ CLERI_MOST_GREEDY,
+ 14,
+ help_access,
+ help_alter,
+ help_count,
+ help_create,
+ help_drop,
+ help_functions,
+ help_grant,
+ help_list,
+ help_noaccess,
+ help_revoke,
+ help_select,
+ help_show,
+ help_timeit,
+ help_timezones
+ ))
+ ));
+
+ cleri_grammar_t * grammar = cleri_grammar(START, "^[a-z_]+");
+
+ return grammar;
+}
--- /dev/null
+/*
+ * health.c
+ */
+#include <siri/health.h>
+#include <siri/siri.h>
+#include <siri/net/tcp.h>
+#include <logger/logger.h>
+
+#define OK_RESPONSE \
+ "HTTP/1.1 200 OK\r\n" \
+ "Content-Type: text/plain\r\n" \
+ "Content-Length: 3\r\n" \
+ "\r\n" \
+ "OK\n"
+
+#define NOK_RESPONSE \
+ "HTTP/1.1 503 Service Unavailable\r\n" \
+ "Content-Type: text/plain\r\n" \
+ "Content-Length: 4\r\n" \
+ "\r\n" \
+ "NOK\n"
+
+#define NFOUND_RESPONSE \
+ "HTTP/1.1 404 Not Found\r\n" \
+ "Content-Type: text/plain\r\n" \
+ "Content-Length: 10\r\n" \
+ "\r\n" \
+ "NOT FOUND\n"
+
+#define MNA_RESPONSE \
+ "HTTP/1.1 405 Method Not Allowed\r\n" \
+ "Content-Type: text/plain\r\n" \
+ "Content-Length: 19\r\n" \
+ "\r\n" \
+ "METHOD NOT ALLOWED\n"
+
+#define SYNC_RESPONSE \
+ "HTTP/1.1 200 OK\r\n" \
+ "Content-Type: text/plain\r\n" \
+ "Content-Length: 14\r\n" \
+ "\r\n" \
+ "SYNCHRONIZING\n"
+
+#define REIDX_RESPONSE \
+ "HTTP/1.1 200 OK\r\n" \
+ "Content-Type: text/plain\r\n" \
+ "Content-Length: 11\r\n" \
+ "\r\n" \
+ "REINDEXING\n"
+
+#define BMODE_RESPONSE \
+ "HTTP/1.1 200 OK\r\n" \
+ "Content-Type: text/plain\r\n" \
+ "Content-Length: 12\r\n" \
+ "\r\n" \
+ "BACKUP MODE\n"
+
+/* static response buffers */
+static uv_buf_t health__uv_ok_buf;
+static uv_buf_t health__uv_nok_buf;
+static uv_buf_t health__uv_nfound_buf;
+static uv_buf_t health__uv_mna_buf;
+static uv_buf_t health__uv_sync_buf;
+static uv_buf_t health__uv_reidx_buf;
+static uv_buf_t health__uv_bmode_buf;
+
+static uv_tcp_t health__uv_server;
+static http_parser_settings health__settings;
+
+static void health__close_cb(uv_handle_t * handle)
+{
+ siri_health_request_t * web_request = handle->data;
+ free(web_request);
+}
+
+static void health__alloc_cb(
+ uv_handle_t * handle __attribute__((unused)),
+ size_t sugsz __attribute__((unused)),
+ uv_buf_t * buf)
+{
+ buf->base = malloc(HTTP_MAX_HEADER_SIZE);
+ buf->len = buf->base ? HTTP_MAX_HEADER_SIZE : 0;
+}
+
+static void health__data_cb(
+ uv_stream_t * uvstream,
+ ssize_t n,
+ const uv_buf_t * buf)
+{
+ size_t parsed;
+ siri_health_request_t * web_request = uvstream->data;
+
+ if (web_request->is_closed)
+ goto done;
+
+ if (n < 0)
+ {
+ if (n != UV_EOF)
+ {
+ log_error(uv_strerror(n));
+ }
+ siri_health_close(web_request);
+ goto done;
+ }
+
+ buf->base[HTTP_MAX_HEADER_SIZE-1] = '\0';
+
+ parsed = http_parser_execute(
+ &web_request->parser,
+ &health__settings,
+ buf->base, n);
+
+ if (web_request->parser.upgrade)
+ {
+ /* TODO: do we need to do something? */
+ log_debug("upgrade to a new protocol");
+ }
+ else if (parsed != (size_t) n)
+ {
+ log_warning("error parsing HTTP request");
+ siri_health_close(web_request);
+ }
+
+done:
+ free(buf->base);
+}
+
+static uv_buf_t * health__get_status_response(void)
+{
+ siridb_t * siridb;
+ uint8_t flags = SERVER_FLAG_RUNNING;
+ llist_node_t * siridb_node;
+
+ siridb_node = siri.siridb_list->first;
+ while (siridb_node != NULL)
+ {
+ siridb = (siridb_t *) siridb_node->data;
+ flags |= siridb->server->flags;
+ siridb_node = siridb_node->next;
+ }
+
+ if (flags & SERVER_FLAG_SYNCHRONIZING)
+ {
+ return &health__uv_sync_buf;
+ }
+ if (flags & SERVER_FLAG_REINDEXING)
+ {
+ return &health__uv_reidx_buf;
+ }
+ if (flags & SERVER_FLAG_BACKUP_MODE)
+ {
+ return &health__uv_bmode_buf;
+ }
+ return flags == SERVER_FLAG_RUNNING
+ ? &health__uv_ok_buf
+ : &health__uv_nok_buf;
+}
+
+static uv_buf_t * health__get_ready_response(void)
+{
+ int status = 1;
+ siridb_t * siridb;
+ llist_node_t * siridb_node;
+
+ siridb_node = siri.siridb_list->first;
+ while (siridb_node != NULL)
+ {
+ siridb = (siridb_t *) siridb_node->data;
+ if (siridb->server->flags != SERVER_FLAG_RUNNING)
+ {
+ status = 0;
+ break;
+ }
+ siridb_node = siridb_node->next;
+ }
+
+ if (status)
+ {
+ return &health__uv_ok_buf;
+ }
+
+ if (siri.args->managed)
+ {
+ /*
+ * If managed, return NOK only if the server is not RUNNING and the
+ * server is not SERVER_FLAG_REINDEXING and the server has either no
+ * replica, or the replica is (maybe) online
+ *
+ * In case the the replica is off-line we want to respond using `OK`
+ * since the an environment like Kubernetes can continue to start
+ * the next pod. (see issue #153)
+ */
+ siridb_node = siri.siridb_list->first;
+ while (siridb_node != NULL)
+ {
+ siridb = (siridb_t *) siridb_node->data;
+ if (siridb->server->flags != SERVER_FLAG_RUNNING &&
+ (~siridb->server->flags & SERVER_FLAG_REINDEXING) && (
+ siridb->replica == NULL ||
+ siridb->replica->retry_attempts < 3))
+ {
+ return &health__uv_nok_buf;
+ }
+ siridb_node = siridb_node->next;
+ }
+ return &health__uv_ok_buf;
+ }
+
+ return &health__uv_nok_buf;
+}
+
+static int health__url_cb(http_parser * parser, const char * at, size_t length)
+{
+ siri_health_request_t * web_request = parser->data;
+
+ web_request->response
+
+ /* status response */
+ = ((length == 1 && *at == '/') ||
+ (length == 7 && memcmp(at, "/status", 7) == 0))
+ ? health__get_status_response()
+
+ /* ready response */
+ : (length == 6 && memcmp(at, "/ready", 6) == 0)
+ ? health__get_ready_response()
+
+ /* healthy response */
+ : (length == 8 && memcmp(at, "/healthy", 8) == 0)
+ ? &health__uv_ok_buf
+
+ /* everything else */
+ : &health__uv_nfound_buf;
+
+ return 0;
+}
+
+static void health__write_cb(uv_write_t * req, int status)
+{
+ if (status)
+ log_error("error writing HTTP response: `%s`", uv_strerror(status));
+
+ siri_health_close((siri_health_request_t *) req->handle->data);
+}
+
+static int health__message_complete_cb(http_parser * parser)
+{
+ siri_health_request_t * web_request = parser->data;
+
+ if (parser->method != HTTP_GET)
+ web_request->response = &health__uv_mna_buf;
+ else if (!web_request->response)
+ web_request->response = &health__uv_nfound_buf;
+
+ (void) uv_write(
+ &web_request->req,
+ &web_request->uvstream,
+ web_request->response, 1,
+ health__write_cb);
+
+ return 0;
+}
+
+static void health__connection_cb(uv_stream_t * server, int status)
+{
+ int rc;
+ siri_health_request_t * web_request;
+
+ if (status < 0)
+ {
+ log_error("HTTP connection error: `%s`", uv_strerror(status));
+ return;
+ }
+
+ log_debug("received a HTTP status connection request");
+
+ web_request = malloc(sizeof(siri_health_request_t));
+ if (!web_request)
+ {
+ ERR_ALLOC
+ return;
+ }
+
+ (void) uv_tcp_init(siri.loop, (uv_tcp_t *) &web_request->uvstream);
+
+ web_request->flags = SIRIDB_HEALTH_FLAG;
+ web_request->is_closed = false;
+ web_request->uvstream.data = web_request;
+ web_request->parser.data = web_request;
+
+ rc = uv_accept(server, &web_request->uvstream);
+ if (rc)
+ {
+ log_error("cannot accept HTTP request: `%s`", uv_strerror(rc));
+ siri_health_close(web_request);
+ return;
+ }
+
+ http_parser_init(&web_request->parser, HTTP_REQUEST);
+
+ rc = uv_read_start(&web_request->uvstream, health__alloc_cb, health__data_cb);
+ if (rc)
+ {
+ log_error("cannot read HTTP request: `%s`", uv_strerror(rc));
+ siri_health_close(web_request);
+ return;
+ }
+}
+
+int siri_health_init(void)
+{
+ int rc;
+ struct sockaddr_storage addr = {0};
+ uint16_t port = siri.cfg->http_status_port;
+
+ if (siri.cfg->ip_support == IP_SUPPORT_IPV4ONLY)
+ {
+ (void) uv_ip4_addr("0.0.0.0", (int) port, (struct sockaddr_in *) &addr);
+ } else
+ {
+ (void) uv_ip6_addr("::", (int) port, (struct sockaddr_in6 *) &addr);
+ }
+
+ health__uv_ok_buf =
+ uv_buf_init(OK_RESPONSE, strlen(OK_RESPONSE));
+ health__uv_nok_buf =
+ uv_buf_init(NOK_RESPONSE, strlen(NOK_RESPONSE));
+ health__uv_nfound_buf =
+ uv_buf_init(NFOUND_RESPONSE, strlen(NFOUND_RESPONSE));
+ health__uv_mna_buf =
+ uv_buf_init(MNA_RESPONSE, strlen(MNA_RESPONSE));
+ health__uv_sync_buf =
+ uv_buf_init(SYNC_RESPONSE, strlen(SYNC_RESPONSE));
+ health__uv_reidx_buf =
+ uv_buf_init(REIDX_RESPONSE, strlen(REIDX_RESPONSE));
+ health__uv_bmode_buf =
+ uv_buf_init(BMODE_RESPONSE, strlen(BMODE_RESPONSE));
+
+ health__settings.on_url = health__url_cb;
+ health__settings.on_message_complete = health__message_complete_cb;
+
+ if (
+ (rc = uv_tcp_init(siri.loop, &health__uv_server)) ||
+ (rc = uv_tcp_bind(
+ &health__uv_server,
+ (const struct sockaddr *) &addr,
+ (siri.cfg->ip_support == IP_SUPPORT_IPV6ONLY) ? UV_TCP_IPV6ONLY : 0)) ||
+ (rc = uv_listen(
+ (uv_stream_t *) &health__uv_server,
+ 128,
+ health__connection_cb)))
+ {
+ log_error("error initializing HTTP status server on port %u: `%s`",
+ port,
+ uv_strerror(rc));
+ return -1;
+ }
+
+ log_info("Start listening for HTTP status requests on TCP port %u", port);
+ return 0;
+}
+
+void siri_health_close(siri_health_request_t * web_request)
+{
+ if (!web_request || web_request->is_closed)
+ return;
+ web_request->is_closed = true;
+ uv_close((uv_handle_t *) &web_request->uvstream, health__close_cb);
+}
--- /dev/null
+/*
+ * heartbeat.c - Heart-beat task SiriDB.
+ *
+ * There is one and only one heart-beat task thread running for SiriDB. For
+ * this reason we do not need to parse data but we should only take care for
+ * locks while writing data.
+ */
+#include <logger/logger.h>
+#include <siri/db/server.h>
+#include <siri/heartbeat.h>
+#include <uv.h>
+
+static uv_timer_t heartbeat;
+
+#define HEARTBEAT_INIT_TIMEOUT 1000
+
+static void HEARTBEAT_cb(uv_timer_t * handle);
+
+void siri_heartbeat_init(siri_t * siri)
+{
+ uint64_t repeat = siri->cfg->heartbeat_interval * 1000;
+ siri->heartbeat = &heartbeat;
+ uv_timer_init(siri->loop, &heartbeat);
+ uv_timer_start(
+ &heartbeat,
+ HEARTBEAT_cb,
+ HEARTBEAT_INIT_TIMEOUT,
+ repeat);
+}
+
+void siri_heartbeat_stop(siri_t * siri)
+{
+ /* stop the timer so it will not run again */
+ uv_timer_stop(&heartbeat);
+ uv_close((uv_handle_t *) &heartbeat, NULL);
+
+ /* we do not need the heart-beat anymore */
+ siri->heartbeat = NULL;
+}
+
+void siri_heartbeat_force(void)
+{
+ HEARTBEAT_cb(NULL);
+}
+
+static void HEARTBEAT_cb(uv_timer_t * handle __attribute__((unused)))
+{
+ siridb_t * siridb;
+ siridb_server_t * server;
+
+ llist_node_t * siridb_node;
+ llist_node_t * server_node;
+
+ log_debug("Start heart-beat task");
+
+ siridb_node = siri.siridb_list->first;
+
+ while (siridb_node != NULL)
+ {
+ siridb = (siridb_t *) siridb_node->data;
+
+ siridb_update_shard_expiration(siridb);
+
+ if ( siridb_tee_is_configured(siridb->tee) &&
+ !siridb_tee_is_connected(siridb->tee))
+ {
+ siridb_tee_connect(siridb->tee);
+ }
+
+ server_node = siridb->servers->first;
+ while (server_node != NULL)
+ {
+ server = (siridb_server_t *) server_node->data;
+
+ if (server != siridb->server && server->client == NULL)
+ {
+ siridb_server_connect(siridb, server);
+ }
+ else if (siridb_server_is_online(server))
+ {
+ siridb_server_send_flags(server);
+ }
+
+ server_node = server_node->next;
+ }
+
+ siridb_node = siridb_node->next;
+ }
+}
+
--- /dev/null
+/*
+ * help.c - Help for SiriDB.
+ */
+#include <limits.h>
+#include <logger/logger.h>
+#include <siri/db/db.h>
+#include <siri/help/help.h>
+#include <stdio.h>
+#include <xpath/xpath.h>
+
+static char * siri_help_content[HELP_COUNT] = {0};
+
+/*
+ * Returns the help mark-down content from cache if possible or otherwise
+ * the help file will be read from disk.
+ *
+ * In case of an error NULL is returned and an error message is set.
+ */
+const char * siri_help_get(
+ uint16_t gid,
+ const char * help_name,
+ char * err_msg)
+{
+ char ** content = &siri_help_content[gid - HELP_OFFSET];
+
+ if (*content == NULL)
+ {
+ /*
+ * path must be initialized for xpath_get_exec_path to handle this variable
+ * correctly.
+ */
+ char path[XPATH_MAX];
+ char fn[XPATH_MAX];
+
+ memset(&path, 0, sizeof(path));
+
+ if (xpath_get_exec_path(path))
+ {
+ sprintf(err_msg, "Error while reading executable path");
+ }
+ else
+ {
+ if (help_name[0] == '?')
+ {
+ if (snprintf(
+ fn,
+ XPATH_MAX,
+ "%shelp/help%s.md",
+ path,
+ help_name + 1) >= XPATH_MAX)
+ {
+ fn[XPATH_MAX-1] = '\0';
+ }
+ }
+ else
+ {
+ if (snprintf(
+ fn,
+ XPATH_MAX,
+ "%shelp/%s.md",
+ path,
+ help_name) >= XPATH_MAX)
+ {
+ fn[XPATH_MAX-1] = '\0';
+ }
+ }
+
+ log_debug("Reading help file: '%s'", fn);
+
+ FILE * fp = fopen (fn, "rb");
+
+ if (fp == NULL)
+ {
+ if (snprintf(
+ err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot open help file: '%s'",
+ fn) >= SIRIDB_MAX_SIZE_ERR_MSG)
+ {
+ err_msg[SIRIDB_MAX_SIZE_ERR_MSG-1] = '\0';
+ }
+ }
+ else
+ {
+ size_t size;
+
+ fseeko(fp, 0, SEEK_END);
+ size = ftello(fp);
+ fseeko(fp, 0, SEEK_SET);
+
+ *content = malloc (size + 1);
+
+ if (*content == NULL)
+ {
+ if (snprintf(
+ err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Memory allocation error while reading help "
+ "file: '%s'",
+ fn) >= SIRIDB_MAX_SIZE_ERR_MSG)
+ {
+ err_msg[SIRIDB_MAX_SIZE_ERR_MSG-1] = '\0';
+ }
+ }
+ else if (fread(*content, 1, size, fp) != size)
+ {
+ if (snprintf(
+ err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error while reading help file: '%s'",
+ fn) >= SIRIDB_MAX_SIZE_ERR_MSG)
+ {
+ err_msg[SIRIDB_MAX_SIZE_ERR_MSG-1] = '\0';
+ }
+ free(*content);
+ *content = NULL;
+ }
+ else
+ {
+ (*content)[size] = '\0';
+ }
+
+ fclose(fp);
+ }
+ }
+ }
+ return *content;
+}
+
+/*
+ * Destroy the help. (free help content in cache)
+ */
+void siri_help_free(void)
+{
+ uint_fast16_t i;
+
+ for (i = 0; i < HELP_COUNT; i ++)
+ {
+ free(siri_help_content[i]);
+ }
+}
+
--- /dev/null
+/*
+ * bserver.c - Listen to back-end SiriDB Server.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/auth.h>
+#include <siri/db/groups.h>
+#include <siri/db/insert.h>
+#include <siri/db/query.h>
+#include <siri/db/replicate.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/net/bserver.h>
+#include <siri/net/pkg.h>
+#include <siri/net/protocol.h>
+#include <siri/net/stream.h>
+#include <siri/net/tcp.h>
+#include <siri/optimize.h>
+#include <siri/siri.h>
+#include <stdlib.h>
+
+#define DEFAULT_BACKLOG 128
+
+#define SERVER_CHECK_AUTHENTICATED(client__, server__) \
+siridb_server_t * server__ = (client__)->origin; \
+if (!(server__->flags & SERVER_FLAG_AUTHENTICATED)) \
+{ \
+ sirinet_pkg_t * package; \
+ package = sirinet_pkg_new( \
+ pkg->pid, 0, BPROTO_ERR_NOT_AUTHENTICATED, NULL); \
+ sirinet_pkg_send(client, package); \
+ return; \
+}
+
+static void BSERVER_flags_update(
+ siridb_t * siridb,
+ siridb_server_t * server,
+ int64_t flags);
+static void on_new_connection(uv_stream_t * server, int status);
+static void on_data(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_auth_request(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_flags_update(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_log_level_update(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg);
+static void on_tee_pipe_name_update(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg);
+static void on_drop_database(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_repl_finished(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_query(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg, int flags);
+static void on_insert(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg, int flags);
+static void on_register_server(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_drop_series(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_req_groups(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_enable_backup_mode(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg);
+static void on_disable_backup_mode(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg);
+static void on_req_tags(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_series_tags(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_empty_tags(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+
+static uv_loop_t * loop = NULL;
+static struct sockaddr_storage server_addr;
+static uv_tcp_t backend_server;
+
+int sirinet_bserver_init(siri_t * siri)
+{
+ assert (loop == NULL);
+
+ int rc;
+ int ip_v6 = 0; /* false */
+ char * ip;
+
+ /* bind loop to the given loop */
+ loop = siri->loop;
+
+ uv_tcp_init(loop, &backend_server);
+
+ /* make sure data is set to NULL so we later on can check this value. */
+ backend_server.data = NULL;
+
+ if (siri->cfg->bind_backend_addr != NULL)
+ {
+ struct in6_addr sa6;
+ if (inet_pton(AF_INET6, siri->cfg->bind_backend_addr, &sa6))
+ {
+ ip_v6 = 1; /* true */
+ }
+ ip = siri->cfg->bind_backend_addr;
+ }
+ else if (siri->cfg->ip_support == IP_SUPPORT_IPV4ONLY)
+ {
+ ip = "0.0.0.0";
+ }
+ else
+ {
+ ip = "::";
+ ip_v6 = 1; /* true */
+ }
+
+ if (ip_v6)
+ {
+ uv_ip6_addr(
+ ip,
+ siri->cfg->listen_backend_port,
+ (struct sockaddr_in6 *) &server_addr);
+ }
+ else
+ {
+ uv_ip4_addr(
+ ip,
+ siri->cfg->listen_backend_port,
+ (struct sockaddr_in *) &server_addr);
+ }
+
+ uv_tcp_bind(
+ &backend_server,
+ (const struct sockaddr *) &server_addr,
+ (siri->cfg->ip_support == IP_SUPPORT_IPV6ONLY) ?
+ UV_TCP_IPV6ONLY : 0);
+
+ rc = uv_listen(
+ (uv_stream_t *) &backend_server,
+ DEFAULT_BACKLOG,
+ on_new_connection);
+
+ if (rc)
+ {
+ log_error("Error listening back-end server: %s", uv_strerror(rc));
+ return 1;
+ }
+
+ log_info("Start listening for back-end connections on port %d",
+ siri->cfg->listen_backend_port);
+
+ return 0;
+}
+
+static void on_new_connection(uv_stream_t * server, int status)
+{
+ if (status < 0)
+ {
+ log_error("Back-end server connection error: %s", uv_strerror(status));
+ return;
+ }
+
+ log_debug("Received a back-end server connection request.");
+
+ sirinet_stream_t * client = sirinet_stream_new(
+ STREAM_TCP_BACKEND, (on_data_cb_t) &on_data);
+
+ if (client != NULL)
+ {
+ uv_tcp_init(loop, (uv_tcp_t *) client->stream);
+
+ if (uv_accept(server, client->stream) == 0)
+ {
+ uv_read_start(
+ client->stream,
+ sirinet_stream_alloc_buffer,
+ sirinet_stream_on_data);
+ }
+ else
+ {
+ sirinet_stream_decref(client);
+ }
+ }
+}
+
+static void BSERVER_flags_update(
+ siridb_t * siridb,
+ siridb_server_t * server,
+ int64_t flags)
+{
+ /* update server flags */
+ siridb_server_update_flags(server->flags, flags);
+
+ if (server->client == NULL)
+ {
+ /* connect in case we do not have a connection yet */
+ siridb_server_connect(siridb, server);
+ }
+
+ /* if server is re-indexing, update status */
+ if ( (siridb->flags & SIRIDB_FLAG_REINDEXING) &&
+ (~siridb->server->flags & SERVER_FLAG_REINDEXING) &&
+ (~server->flags & SERVER_FLAG_REINDEXING))
+ {
+ siridb_reindex_status_update(siridb);
+ }
+
+ /* if this is the replica see if we have anything to update */
+ if ( siridb->replica == server &&
+ siridb_replicate_is_idle(siridb->replicate))
+ {
+ siridb_replicate_start(siridb->replicate);
+ }
+
+ log_info("Status received from '%s' (status: %d)",
+ server->name,
+ server->flags);
+}
+
+static void on_data(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ if (Logger.level == LOGGER_DEBUG)
+ {
+ char * name = sirinet_stream_name(client);
+ if (name != NULL)
+ {
+ log_debug(
+ "Package received from server '%s' "
+ "(pid: %" PRIu16 ", len: %" PRIu32 ", tp: %s)",
+ name,
+ pkg->pid,
+ pkg->len,
+ sirinet_bproto_client_str(pkg->tp));
+ free(name);
+ }
+ }
+
+ switch ((bproto_client_t) pkg->tp)
+ {
+ case BPROTO_AUTH_REQUEST:
+ on_auth_request(client, pkg);
+ break;
+ case BPROTO_FLAGS_UPDATE:
+ on_flags_update(client, pkg);
+ break;
+ case BPROTO_LOG_LEVEL_UPDATE:
+ on_log_level_update(client, pkg);
+ break;
+ case BPROTO_REPL_FINISHED:
+ on_repl_finished(client, pkg);
+ break;
+ case BPROTO_QUERY_SERVER:
+ on_query(client, pkg, 0);
+ break;
+ case BPROTO_QUERY_UPDATE:
+ on_query(client, pkg, SIRIDB_QUERY_FLAG_UPDATE_REPLICA);
+ break;
+ case BPROTO_INSERT_POOL:
+ on_insert(client, pkg, INSERT_FLAG_POOL);
+ break;
+ case BPROTO_INSERT_SERVER:
+ on_insert(client, pkg, 0);
+ break;
+ case BPROTO_INSERT_TEST_POOL:
+ on_insert(client, pkg, INSERT_FLAG_POOL | INSERT_FLAG_TEST);
+ break;
+ case BPROTO_INSERT_TEST_SERVER:
+ on_insert(client, pkg, INSERT_FLAG_TEST);
+ break;
+ case BPROTO_INSERT_TESTED_POOL:
+ on_insert(client, pkg, INSERT_FLAG_POOL | INSERT_FLAG_TESTED);
+ break;
+ case BPROTO_INSERT_TESTED_SERVER:
+ on_insert(client, pkg, INSERT_FLAG_TESTED);
+ break;
+ case BPROTO_REGISTER_SERVER:
+ on_register_server(client, pkg);
+ break;
+ case BPROTO_DROP_SERIES:
+ on_drop_series(client, pkg);
+ break;
+ case BPROTO_REQ_GROUPS:
+ on_req_groups(client, pkg);
+ break;
+ case BPROTO_ENABLE_BACKUP_MODE:
+ on_enable_backup_mode(client, pkg);
+ break;
+ case BPROTO_DISABLE_BACKUP_MODE:
+ on_disable_backup_mode(client, pkg);
+ break;
+ case BPROTO_TEE_PIPE_NAME_UPDATE:
+ on_tee_pipe_name_update(client, pkg);
+ break;
+ case BPROTO_DROP_DATABASE:
+ on_drop_database(client, pkg);
+ break;
+ case BPROTO_REQ_TAGS:
+ on_req_tags(client, pkg);
+ break;
+ case BPROTO_SERIES_TAGS:
+ on_series_tags(client, pkg);
+ break;
+ case BPROTO_EMPTY_TAGS:
+ on_empty_tags(client, pkg);
+ break;
+ }
+
+}
+
+static void on_auth_request(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ bproto_server_t rc;
+ sirinet_pkg_t * package;
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_obj_t qp_uuid;
+ qp_obj_t qp_dbname;
+ qp_obj_t qp_flags;
+ qp_obj_t qp_version;
+ qp_obj_t qp_min_version;
+ qp_obj_t qp_ip_support;
+ qp_obj_t qp_libuv;
+ qp_obj_t qp_dbpath;
+ qp_obj_t qp_buffer_path;
+ qp_obj_t qp_buffer_size;
+ qp_obj_t qp_startup_time;
+ qp_obj_t qp_address;
+ qp_obj_t qp_port;
+
+ if ( qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_uuid) == QP_RAW &&
+ qp_next(&unpacker, &qp_dbname) == QP_RAW &&
+ qp_is_raw_term(&qp_dbname) &&
+ qp_next(&unpacker, &qp_flags) == QP_INT64 &&
+ qp_next(&unpacker, &qp_version) == QP_RAW &&
+ qp_is_raw_term(&qp_version) &&
+ qp_next(&unpacker, &qp_min_version) == QP_RAW &&
+ qp_is_raw_term(&qp_min_version) &&
+ qp_next(&unpacker, &qp_ip_support) == QP_INT64 &&
+ qp_next(&unpacker, &qp_libuv) == QP_RAW &&
+ qp_is_raw_term(&qp_libuv) &&
+ qp_next(&unpacker, &qp_dbpath) == QP_RAW &&
+ qp_is_raw_term(&qp_dbpath) &&
+ qp_next(&unpacker, &qp_buffer_path) == QP_RAW &&
+ qp_is_raw_term(&qp_buffer_path) &&
+ qp_next(&unpacker, &qp_buffer_size) == QP_INT64 &&
+ qp_next(&unpacker, &qp_startup_time) == QP_INT64 &&
+ qp_next(&unpacker, &qp_address) == QP_RAW &&
+ qp_is_raw_term(&qp_address) &&
+ qp_next(&unpacker, &qp_port) == QP_INT64)
+ {
+ rc = siridb_auth_server_request(
+ client,
+ &qp_uuid,
+ &qp_dbname,
+ &qp_version,
+ &qp_min_version);
+ if (rc == BPROTO_AUTH_SUCCESS)
+ {
+ siridb_server_t * server = client->origin;
+ siridb_t * siridb = client->siridb;
+
+ /* check and update flags */
+ BSERVER_flags_update(siridb, server, qp_flags.via.int64);
+
+ /* update server address if needed */
+ siridb_server_update_address(
+ siridb,
+ server,
+ (const char *) qp_address.via.raw,
+ (uint16_t) qp_port.via.int64);
+
+ /* update other server properties */
+ free(server->dbpath);
+ server->dbpath = strdup((const char *) qp_dbpath.via.raw);
+ free(server->buffer_path);
+ server->buffer_path =
+ strdup((const char *) qp_buffer_path.via.raw);
+ free(server->libuv);
+ server->libuv = strdup((const char *) qp_libuv.via.raw);
+ server->buffer_size = (size_t) qp_buffer_size.via.int64;
+ server->startup_time = (uint32_t) qp_startup_time.via.int64;
+ server->ip_support = (uint8_t) qp_ip_support.via.int64;
+
+ log_info("Accepting back-end server connection: '%s'",
+ server->name);
+ }
+ else
+ {
+ log_warning("Refusing back-end connection (error code: %d)", rc);
+ }
+
+ package = sirinet_pkg_new(pkg->pid, 0, rc, NULL);
+
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+ else
+ {
+ log_error("Invalid back-end 'on_auth_request' received.");
+ }
+}
+
+static void on_flags_update(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ sirinet_pkg_t * package;
+ siridb_t * siridb = client->siridb;
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_obj_t qp_flags;
+
+ if (qp_next(&unpacker, &qp_flags) == QP_INT64)
+ {
+ /* check and update flags */
+ BSERVER_flags_update(siridb, server, qp_flags.via.int64);
+
+ package = sirinet_pkg_new(pkg->pid, 0, BPROTO_ACK_FLAGS, NULL);
+
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+ else
+ {
+ log_error("Invalid back-end 'on_flags_update' received.");
+ }
+}
+
+static void on_log_level_update(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ sirinet_pkg_t * package;
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_obj_t qp_log_level;
+
+ if (qp_next(&unpacker, &qp_log_level) == QP_INT64)
+ {
+ /* update log level */
+ logger_set_level(qp_log_level.via.int64);
+
+ log_info("Log level update received from '%s' (%s)",
+ server->name,
+ Logger.level_name);
+
+ package = sirinet_pkg_new(pkg->pid, 0, BPROTO_ACK_LOG_LEVEL, NULL);
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+ }
+ else
+ {
+ log_error("Invalid back-end 'on_log_level_update' received.");
+ }
+}
+
+static void on_tee_pipe_name_update(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server);
+ siridb_t * siridb = client->siridb;
+ sirinet_pkg_t * package;
+
+ char * pipe_name = pkg->len
+ ? strndup((const char *) pkg->data, pkg->len)
+ : NULL;
+
+ (void) siridb_tee_set_pipe_name(siridb->tee, pipe_name);
+
+ free(pipe_name);
+
+ package = sirinet_pkg_new(pkg->pid, 0, BPROTO_ACK_TEE_PIPE_NAME, NULL);
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_drop_database(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ siridb_t * siridb = client->siridb;
+ sirinet_pkg_t * package = NULL;
+
+ siridb_drop(siridb);
+
+ package = sirinet_pkg_new(pkg->pid, 0, BPROTO_ACK_DROP_DATABASE, NULL);
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_repl_finished(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ siridb_t * siridb = client->siridb;
+ sirinet_pkg_t * package;
+
+ if (siridb->server->flags & SERVER_FLAG_SYNCHRONIZING)
+ {
+ /* remove synchronization flag */
+ siridb->server->flags &= ~SERVER_FLAG_SYNCHRONIZING;
+
+ /* continue optimize */
+ siri_optimize_continue();
+
+ /* start re-index task if needed */
+ if (siridb->reindex != NULL)
+ {
+ siridb_reindex_start(siridb->reindex->timer);
+ }
+
+ siridb_servers_send_flags(siridb->servers);
+ }
+
+ package = sirinet_pkg_new(pkg->pid, 0, BPROTO_ACK_REPL_FINISHED, NULL);
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_query(sirinet_stream_t * client, sirinet_pkg_t * pkg, int flags)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ qp_obj_t qp_query;
+
+ if (flags & SIRIDB_QUERY_FLAG_UPDATE_REPLICA)
+ {
+ siridb_t * siridb = client->siridb;
+ if (siridb->replica != NULL)
+ {
+ pkg->tp = BPROTO_QUERY_SERVER;
+ siridb_replicate_pkg(siridb, pkg);
+ }
+ }
+
+ if ( qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_query) == QP_RAW)
+ {
+ siridb_query_run(
+ pkg->pid,
+ client,
+ (const char *) qp_query.via.raw,
+ qp_query.len,
+ 0.0,
+ 0);
+ }
+ else
+ {
+ log_error("Invalid back-end 'on_query_server' received.");
+ }
+}
+
+static void on_insert(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg,
+ int flags)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ sirinet_pkg_t * package;
+ siridb_t * siridb = client->siridb;
+ if ((flags & INSERT_FLAG_POOL) && siridb->replica != NULL)
+ {
+ assert (siridb->fifo != NULL);
+ sirinet_pkg_t * repl_pkg;
+
+ if (siridb->replicate->initsync == NULL)
+ {
+ pkg->tp = (flags & INSERT_FLAG_TEST) ?
+ BPROTO_INSERT_TEST_SERVER : (flags & INSERT_FLAG_TESTED) ?
+ BPROTO_INSERT_TESTED_SERVER : BPROTO_INSERT_SERVER;
+ repl_pkg = pkg;
+ }
+ else
+ {
+ repl_pkg = siridb_replicate_pkg_filter(
+ siridb,
+ pkg->data,
+ pkg->len,
+ flags);
+ }
+
+ if (repl_pkg == NULL || siridb_replicate_pkg(siridb, repl_pkg))
+ {
+ /* signal is raised */
+ sirinet_pkg_t * package =
+ sirinet_pkg_new(pkg->pid, 0, BPROTO_ERR_INSERT, NULL);
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+ }
+
+ if (repl_pkg != pkg)
+ {
+ free(repl_pkg);
+ }
+ }
+
+ if (!siri_err)
+ {
+ /*
+ * We do not check here for backup mode since we want back-end inserts
+ * to finish. We rather prevent servers from starting new insert tasks.
+ */
+ if ( (~siridb->server->flags & SERVER_FLAG_RUNNING) ||
+ insert_init_backend_local(siridb, client, pkg, flags))
+ {
+ package = sirinet_pkg_new(pkg->pid, 0, BPROTO_ERR_INSERT, NULL);
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+ }
+ }
+}
+
+static void on_register_server(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ sirinet_pkg_t * package;
+ siridb_t * siridb = client->siridb;
+ siridb_server_t * new_server;
+
+ if ( siridb->server->flags != SERVER_FLAG_RUNNING ||
+ (siridb->flags & SIRIDB_FLAG_REINDEXING))
+ {
+ log_error("Cannot register new server because of having status %d",
+ siridb->server->flags);
+
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ERR_REGISTER_SERVER,
+ NULL);
+ }
+ else
+ {
+ new_server = siridb_server_register(siridb, pkg->data, pkg->len);
+ package = (new_server == NULL) ?
+ /* a signal might be raised */
+ sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ERR_REGISTER_SERVER,
+ NULL) :
+ sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ACK_REGISTER_SERVER,
+ NULL);
+ }
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_drop_series(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ sirinet_pkg_t * package = NULL;
+ siridb_t * siridb = client->siridb;
+
+ if ( (~siridb->server->flags & SERVER_FLAG_RUNNING) ||
+ (siridb->server->flags & SERVER_FLAG_BACKUP_MODE))
+ {
+ log_error("Cannot drop series because of having status %d",
+ siridb->server->flags);
+
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ERR_DROP_SERIES,
+ NULL);
+ }
+ else
+ {
+ qp_obj_t qp_series_name;
+
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_next(&unpacker, &qp_series_name);
+ if (qp_is_raw_term(&qp_series_name))
+ {
+ siridb_series_t * series;
+ series = ct_get(
+ siridb->series,
+ (const char *) qp_series_name.via.raw);
+ if (series != NULL)
+ {
+ uv_mutex_lock(&siridb->series_mutex);
+
+ siridb_series_drop(siridb, series);
+
+ uv_mutex_unlock(&siridb->series_mutex);
+
+ siridb_series_flush_dropped(siridb);
+ }
+ else
+ {
+ log_warning(
+ "Received a request to drop series '%s' but "
+ "the series is not found (already dropped?)",
+ qp_series_name.via.raw);
+ }
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ACK_DROP_SERIES,
+ NULL);
+ }
+ else
+ {
+ log_error(
+ "Illegal back-end dropped series package "
+ "received, probably the series name was not "
+ "terminated?");
+ }
+ }
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_req_groups(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ siridb_t * siridb = client->siridb;
+ sirinet_pkg_t * package = siridb_groups_pkg(siridb->groups, pkg->pid);
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_enable_backup_mode(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ siridb_t * siridb = client->siridb;
+ sirinet_pkg_t * package;
+
+ if ( (~siridb->server->flags & SERVER_FLAG_RUNNING) ||
+ (siridb->flags & SIRIDB_FLAG_REINDEXING) ||
+ (siridb->server->flags & SERVER_FLAG_BACKUP_MODE))
+ {
+ log_error(
+ "Cannot set server '%s' in backup mode because of "
+ "having status %d",
+ siridb->server->name,
+ siridb->server->flags);
+
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ERR_ENABLE_BACKUP_MODE,
+ NULL);
+ }
+ else
+ {
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ (siri_backup_enable(&siri, siridb)) ?
+ BPROTO_ERR_ENABLE_BACKUP_MODE :
+ BPROTO_ACK_ENABLE_BACKUP_MODE,
+ NULL);
+ }
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+
+}
+
+static void on_disable_backup_mode(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ siridb_t * siridb = client->siridb;
+ sirinet_pkg_t * package = NULL;
+
+ if ( (~siridb->server->flags & SERVER_FLAG_RUNNING) ||
+ (siridb->flags & SIRIDB_FLAG_REINDEXING) ||
+ (~siridb->server->flags & SERVER_FLAG_BACKUP_MODE))
+ {
+ log_error(
+ "Cannot clear server '%s' from backup mode because of "
+ "having status %d",
+ siridb->server->name,
+ siridb->server->flags);
+
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ERR_DISABLE_BACKUP_MODE,
+ NULL);
+ }
+ else
+ {
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ (siri_backup_disable(&siri, siridb)) ?
+ BPROTO_ERR_DISABLE_BACKUP_MODE :
+ BPROTO_ACK_DISABLE_BACKUP_MODE,
+ NULL);
+ }
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_req_tags(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ siridb_t * siridb = client->siridb;
+ sirinet_pkg_t * package = siridb_tags_pkg(siridb->tags, pkg->pid);
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_series_tags(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ sirinet_pkg_t * package = NULL;
+ siridb_t * siridb = client->siridb;
+
+ if (~siridb->server->flags & SERVER_FLAG_RUNNING)
+ {
+ log_error("Cannot tag series because of having status %d",
+ siridb->server->flags);
+
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ERR_DROP_SERIES,
+ NULL);
+ }
+ else
+ {
+ qp_obj_t qp_series_name;
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if (qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_series_name) == QP_RAW &&
+ qp_is_raw_term(&qp_series_name))
+ {
+ siridb_series_t * series;
+
+ series = ct_get(
+ siridb->series,
+ (const char *) qp_series_name.via.raw);
+
+ if (series != NULL)
+ {
+ qp_obj_t qp_tag_name;
+
+ /* take a reference since this task might wait for a lock */
+ siridb_series_incref(series);
+
+ uv_mutex_lock(&siridb->tags->mutex);
+
+ while (qp_next(&unpacker, &qp_tag_name) == QP_RAW)
+ {
+ siridb_tag_t * tag = ct_getn(
+ siridb->tags->tags,
+ qp_tag_name.via.str,
+ qp_tag_name.len);
+
+ if (tag == NULL)
+ {
+ tag = siridb_tags_add_n(
+ siridb->tags,
+ qp_tag_name.via.str,
+ qp_tag_name.len);
+ }
+
+ if (tag && imap_add(tag->series, series->id, series) == 0)
+ {
+ siridb_series_incref(series);
+ }
+
+ siridb_tags_set_require_save(siridb->tags, tag);
+ }
+
+ uv_mutex_unlock(&siridb->tags->mutex);
+
+ siridb_series_decref(series);
+ }
+ else
+ {
+ log_warning(
+ "Received a request to tag series '%s' but "
+ "the series is not found (already dropped?)",
+ qp_series_name.via.raw);
+ }
+
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ACK_SERIES_TAGS,
+ NULL);
+ }
+ else
+ {
+ log_error(
+ "Illegal back-end tag series package "
+ "received, probably the series name was not "
+ "terminated?");
+ }
+ }
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
+
+static void on_empty_tags(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ SERVER_CHECK_AUTHENTICATED(client, server)
+
+ sirinet_pkg_t * package = NULL;
+ siridb_t * siridb = client->siridb;
+
+ if (~siridb->server->flags & SERVER_FLAG_RUNNING)
+ {
+ log_error("Cannot tag series because of having status %d",
+ siridb->server->flags);
+
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ERR_DROP_SERIES,
+ NULL);
+ }
+ else
+ {
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if (qp_is_array(qp_next(&unpacker, NULL)))
+ {
+ qp_obj_t qp_tag_name;
+
+ uv_mutex_lock(&siridb->tags->mutex);
+
+ while (qp_next(&unpacker, &qp_tag_name) == QP_RAW)
+ {
+ siridb_tag_t * tag = ct_getn(
+ siridb->tags->tags,
+ qp_tag_name.via.str,
+ qp_tag_name.len);
+
+ if (tag == NULL)
+ {
+ tag = siridb_tags_add_n(
+ siridb->tags,
+ qp_tag_name.via.str,
+ qp_tag_name.len);
+
+ siridb_tags_set_require_save(siridb->tags, tag);
+ }
+ }
+
+ uv_mutex_unlock(&siridb->tags->mutex);
+
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ BPROTO_ACK_EMPTY_TAGS,
+ NULL);
+ }
+ else
+ {
+ log_error("Illegal back-end empty tags package received");
+ }
+ }
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
--- /dev/null
+/*
+ * clserver.c - Listen for client requests.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <lock/lock.h>
+#include <math.h>
+#include <logger/logger.h>
+#include <qpack/qpack.h>
+#include <siri/siri.h>
+#include <siri/service/account.h>
+#include <siri/service/request.h>
+#include <siri/db/access.h>
+#include <siri/db/auth.h>
+#include <siri/db/insert.h>
+#include <siri/db/query.h>
+#include <siri/db/replicate.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/db/users.h>
+#include <siri/err.h>
+#include <siri/net/clserver.h>
+#include <siri/net/promises.h>
+#include <siri/net/protocol.h>
+#include <siri/net/tcp.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+const unsigned long int WARNING_PKG_SIZE = RESET_BUF_SIZE;
+
+/*
+ * note: this size is chosen to be 65535 but is not restricted to 16bit and
+ * can be larger if required.
+ */
+#define MAX_QUERY_PKG_SIZE 65535
+
+#define DEFAULT_BACKLOG 128
+#define CHECK_SIRIDB(client__, siridb__) \
+siridb_t * siridb__ = (client__)->siridb; \
+if (siridb__ == NULL) \
+{ \
+ sirinet_pkg_t * package; \
+ package = sirinet_pkg_new( \
+ pkg->pid, 0, CPROTO_ERR_NOT_AUTHENTICATED, NULL); \
+ if (package != NULL) \
+ { \
+ sirinet_pkg_send(client, package); \
+ } \
+ return; \
+}
+
+static const int SERVER_RUNNING_REINDEXING =
+ SERVER_FLAG_RUNNING + SERVER_FLAG_REINDEXING;
+
+static uv_loop_t * loop = NULL;
+static struct sockaddr_storage client_addr;
+static uv_tcp_t client_server_tcp;
+static uv_pipe_t client_server_pipe;
+
+static void on_tcp_new_connection(uv_stream_t * server, int status);
+static void on_pipe_new_connection(uv_stream_t * server, int status);
+static void on_data(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_stream_data(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_auth_request(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_query(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_insert(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_ping(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+
+static void on_reqfile(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg,
+ sirinet_clserver_getfile getfile);
+static void on_register_server(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void on_req_service(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void CLSERVER_send_server_error(
+ siridb_t * siridb,
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg);
+static void CLSERVER_send_pool_error(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg);
+static void CLSERVER_on_register_server_response(
+ vec_t * promises,
+ siridb_server_async_t * server_reg);
+
+#define POOL_ERR_MSG \
+ "At least one pool has no server available to process the request"
+
+#define POOL_ERR_LEN 64 /* exact length of POOL_ERR_MSG */
+
+
+int sirinet_clserver_init(siri_t * siri)
+{
+ assert (loop == NULL && siri->loop != NULL);
+
+ int rc;
+ int ip_v6 = 0; /* false */
+ char * ip;
+
+ /* bind loop to the given loop */
+ loop = siri->loop;
+
+ uv_tcp_init(loop, &client_server_tcp);
+ uv_pipe_init(loop, &client_server_pipe, 0);
+
+ /* make sure data is set to NULL so we later on can check this value. */
+ client_server_tcp.data = NULL;
+ client_server_pipe.data = NULL;
+
+ if (siri->cfg->bind_client_addr != NULL)
+ {
+ struct in6_addr sa6;
+ if (inet_pton(AF_INET6, siri->cfg->bind_client_addr, &sa6))
+ {
+ ip_v6 = 1; /* true */
+ }
+ ip = siri->cfg->bind_client_addr;
+ }
+ else if (siri->cfg->ip_support == IP_SUPPORT_IPV4ONLY)
+ {
+ ip = "0.0.0.0";
+ }
+ else
+ {
+ ip = "::";
+ ip_v6 = 1; /* true */
+ }
+
+ if (ip_v6)
+ {
+ uv_ip6_addr(
+ ip,
+ siri->cfg->listen_client_port,
+ (struct sockaddr_in6 *) &client_addr);
+ }
+ else
+ {
+ uv_ip4_addr(
+ ip,
+ siri->cfg->listen_client_port,
+ (struct sockaddr_in *) &client_addr);
+ }
+
+ rc = uv_tcp_bind(
+ &client_server_tcp,
+ (const struct sockaddr *) &client_addr,
+ (siri->cfg->ip_support == IP_SUPPORT_IPV6ONLY) ?
+ UV_TCP_IPV6ONLY : 0);
+
+ if (rc)
+ {
+ log_error("Error binding TCP client server: %s", uv_strerror(rc));
+ return 1;
+ }
+
+ rc = uv_listen(
+ (uv_stream_t *) &client_server_tcp,
+ DEFAULT_BACKLOG,
+ on_tcp_new_connection);
+
+ if (rc)
+ {
+ log_error("Error listening TCP client server: %s", uv_strerror(rc));
+ return 1;
+ }
+
+ log_info("Start listening for TCP client connections on port %d",
+ siri->cfg->listen_client_port);
+
+ if (siri->cfg->pipe_support)
+ {
+ char * pipe_name = siri->cfg->pipe_client_name;
+
+ rc = uv_pipe_bind(&client_server_pipe, pipe_name);
+
+ if (rc)
+ {
+ log_error("Error binding pipe client server: %s", uv_strerror(rc));
+ return 1;
+ }
+
+ rc = uv_listen(
+ (uv_stream_t *) &client_server_pipe,
+ DEFAULT_BACKLOG,
+ on_pipe_new_connection);
+
+ if (rc)
+ {
+ log_error(
+ "Error listening pipe client server: %s", uv_strerror(rc));
+ return 1;
+ }
+
+ log_info("Start listening for pipe client connections on '%s'",
+ pipe_name);
+ }
+
+ return 0;
+}
+
+static void on_tcp_new_connection(uv_stream_t * server, int status)
+{
+ log_debug("Received a TCP client connection request.");
+
+ if (status < 0)
+ {
+ log_error("TCP client connection error: %s", uv_strerror(status));
+ return;
+ }
+ sirinet_stream_t * client = sirinet_stream_new(
+ STREAM_TCP_CLIENT, (on_data_cb_t) &on_stream_data);
+
+ if (client != NULL)
+ {
+ uv_tcp_init(loop, (uv_tcp_t *) client->stream);
+
+ if (uv_accept(server, client->stream) == 0)
+ {
+ uv_read_start(
+ client->stream,
+ sirinet_stream_alloc_buffer,
+ sirinet_stream_on_data);
+ }
+ else
+ {
+ sirinet_stream_decref(client);
+ }
+ }
+}
+
+static void on_pipe_new_connection(uv_stream_t * server, int status)
+{
+ log_debug("Received a pipe client connection request.");
+
+ if (status < 0)
+ {
+ log_error("Pipe client connection error: %s", uv_strerror(status));
+ return;
+ }
+ sirinet_stream_t * client = sirinet_stream_new(
+ STREAM_PIPE_CLIENT, (on_data_cb_t) &on_stream_data);
+
+ if (client != NULL)
+ {
+ uv_pipe_init(loop, (uv_pipe_t *) client->stream, 0);
+
+ if (uv_accept(server,client->stream) == 0)
+ {
+ uv_read_start(
+ client->stream,
+ sirinet_stream_alloc_buffer,
+ sirinet_stream_on_data);
+ }
+ else
+ {
+ sirinet_stream_decref(client);
+ }
+ }
+}
+
+static void on_data(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ /* in case the online flag is not set, we cannot perform any request */
+ if (siri.status == SIRI_STATUS_RUNNING)
+ {
+ switch ((cproto_client_t) pkg->tp)
+ {
+ case CPROTO_REQ_QUERY:
+ on_query(client, pkg);
+ break;
+ case CPROTO_REQ_INSERT:
+ on_insert(client, pkg);
+ break;
+ case CPROTO_REQ_AUTH:
+ on_auth_request(client, pkg);
+ break;
+ case CPROTO_REQ_PING:
+ on_ping(client, pkg);
+ break;
+ case CPROTO_REQ_REGISTER_SERVER:
+ on_register_server(client, pkg);
+ break;
+ case CPROTO_REQ_FILE_SERVERS:
+ on_reqfile(client, pkg, siridb_servers_get_file);
+ break;
+ case CPROTO_REQ_FILE_USERS:
+ on_reqfile(client, pkg, siridb_users_get_file);
+ break;
+ case CPROTO_REQ_FILE_GROUPS:
+ on_reqfile(client, pkg, siridb_groups_get_file);
+ break;
+ case CPROTO_REQ_FILE_DATABASE:
+ on_reqfile(client, pkg, siridb_get_file);
+ break;
+ case CPROTO_REQ_SERVICE:
+ on_req_service(client, pkg);
+ break;
+ }
+ }
+ else
+ {
+ /* siridb can be NULL here, make sure we can handle this state */
+ CLSERVER_send_server_error(client->siridb, client, pkg);
+ }
+}
+
+static void on_stream_data(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ if (Logger.level == LOGGER_DEBUG)
+ {
+ char * name = sirinet_stream_name(client);
+ if (name != NULL)
+ {
+ log_debug(
+ "Package received from client '%s' "
+ "(pid: %" PRIu16 ", len: %" PRIu32 ", tp: %s)",
+ name,
+ pkg->pid,
+ pkg->len,
+ sirinet_cproto_client_str(pkg->tp));
+ free(name);
+ }
+ }
+ else if (pkg->len >= WARNING_PKG_SIZE)
+ {
+ char * name = sirinet_stream_name(client);
+ if (name != NULL)
+ {
+ log_warning(
+ "Got a large package from '%s' (pid: %d, len: %d, tp: %s)."
+ " A package size smaller than 1MB is recommended!",
+ name,
+ pkg->pid,
+ pkg->len,
+ sirinet_cproto_client_str(pkg->tp));
+ free(name);
+ }
+ }
+
+ on_data(client, pkg);
+}
+
+static void on_auth_request(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ cproto_server_t rc;
+ sirinet_pkg_t * package;
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_obj_t qp_username;
+ qp_obj_t qp_password;
+ qp_obj_t qp_dbname;
+
+ if ( qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_username) == QP_RAW &&
+ qp_next(&unpacker, &qp_password) == QP_RAW &&
+ qp_next(&unpacker, &qp_dbname) == QP_RAW)
+ {
+ rc = siridb_auth_user_request(
+ client,
+ &qp_username,
+ &qp_password,
+ &qp_dbname);
+ package = sirinet_pkg_new(pkg->pid, 0, rc, NULL);
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+ }
+ else
+ {
+ log_error("Invalid 'on_auth_request' received.");
+ }
+}
+
+/*
+ * A signal is raised in case an allocation error occurred.
+ */
+static void CLSERVER_send_server_error(
+ siridb_t * siridb,
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg)
+{
+ /* WARNING: siridb can be NULL here */
+
+ char * err_msg;
+ int len = (siridb == NULL) ?
+ asprintf(
+ &err_msg,
+ "Not accepting the request because the sever is not running")
+ :
+ asprintf(
+ &err_msg,
+ "Server '%s' is not accepting the request because of having "
+ "status: %u",
+ siridb->server->name,
+ siridb->server->flags);
+
+ if (len < 0)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ log_debug(err_msg);
+
+ sirinet_pkg_t * package = sirinet_pkg_err(
+ pkg->pid,
+ len,
+ CPROTO_ERR_SERVER,
+ err_msg);
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+ free(err_msg);
+ }
+}
+
+/*
+ * A signal is raised in case an allocation error occurred.
+ */
+static void CLSERVER_send_pool_error(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg)
+{
+ log_debug(POOL_ERR_MSG);
+
+ sirinet_pkg_t * package = sirinet_pkg_err(
+ pkg->pid,
+ POOL_ERR_LEN,
+ CPROTO_ERR_POOL,
+ POOL_ERR_MSG);
+
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+
+ }
+}
+
+static void on_query(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ CHECK_SIRIDB(client, siridb)
+
+ if (pkg->len > MAX_QUERY_PKG_SIZE)
+ {
+ sirinet_pkg_t * package;
+
+ log_error(
+ "Received a query exceeding the maximum size. "
+ "(%" PRIu32 ", max size: %u)",
+ pkg->len,
+ MAX_QUERY_PKG_SIZE);
+
+ package = sirinet_pkg_err(
+ pkg->pid,
+ 15,
+ CPROTO_ERR_QUERY,
+ "Query too long.");
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+
+ return;
+ }
+
+ qp_unpacker_t unpacker;
+ qp_obj_t qp_query;
+ qp_obj_t qp_time_precision;
+ float factor;
+ siridb_timep_t tp = SIRIDB_TIME_DEFAULT;
+
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if ( qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_query) == QP_RAW)
+ {
+ qp_next(&unpacker, &qp_time_precision);
+
+ if (qp_time_precision.tp == QP_INT64 &&
+ (tp = (siridb_timep_t) qp_time_precision.via.int64) !=
+ siridb->time->precision)
+ {
+ tp %= SIRIDB_TIME_END;
+ }
+ factor = (tp == SIRIDB_TIME_DEFAULT) ? 0.0 :
+ pow(1000.0, tp - siridb->time->precision);
+
+ siridb_query_run(
+ pkg->pid,
+ client,
+ (const char *) qp_query.via.raw,
+ qp_query.len,
+ factor,
+ SIRIDB_QUERY_FLAG_MASTER);
+ }
+ else
+ {
+ log_error("Incorrect package received: 'on_query'");
+ }
+}
+
+static void on_insert(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ CHECK_SIRIDB(client, siridb)
+
+ char err_msg[SIRIDB_MAX_SIZE_ERR_MSG];
+
+ if (!siridb_user_check_access(
+ (siridb_user_t *) client->origin,
+ SIRIDB_ACCESS_INSERT,
+ err_msg))
+ {
+ log_warning("(%s) %s",
+ sirinet_cproto_server_str(CPROTO_ERR_USER_ACCESS),
+ err_msg);
+ sirinet_pkg_t * package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_USER_ACCESS,
+ err_msg);
+
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+
+ return;
+ }
+
+ /* only when when the flag is EXACTLY running or
+ * running + re-indexing we can continue */
+ if ( siridb->server->flags != SERVER_FLAG_RUNNING &&
+ siridb->server->flags != SERVER_RUNNING_REINDEXING)
+ {
+ CLSERVER_send_server_error(siridb, client, pkg);
+ return;
+ }
+
+ if (!siridb_pools_accessible(siridb))
+ {
+ CLSERVER_send_pool_error(client, pkg);
+ return;
+ }
+
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ siridb_insert_t * insert = siridb_insert_new(
+ siridb,
+ pkg->pid,
+ client);
+
+ if (insert != NULL)
+ {
+ ssize_t rc = siridb_insert_assign_pools(
+ siridb,
+ &unpacker,
+ insert->packer);
+
+ switch ((siridb_insert_err_t) rc)
+ {
+ case ERR_EXPECTING_ARRAY:
+ case ERR_EXPECTING_SERIES_NAME:
+ case ERR_EXPECTING_MAP_OR_ARRAY:
+ case ERR_EXPECTING_INTEGER_TS:
+ case ERR_TIMESTAMP_OUT_OF_RANGE:
+ case ERR_UNSUPPORTED_VALUE:
+ case ERR_EXPECTING_AT_LEAST_ONE_POINT:
+ case ERR_EXPECTING_NAME_AND_POINTS:
+ case ERR_INCOMPATIBLE_SERVER_VERSION:
+ case ERR_MEM_ALLOC:
+ {
+ /* something went wrong, get correct err message */
+ const char * err_msg = siridb_insert_err_msg(rc);
+
+ log_error("Insert error: '%s' at position %lu",
+ err_msg, unpacker.pt - pkg->data);
+
+ /* create and send package */
+ sirinet_pkg_t * package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_INSERT,
+ err_msg);
+
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+ }
+
+ /* error, free insert */
+ siridb_insert_free(insert);
+ break;
+
+ default:
+ if (siridb_insert_points_to_pools(insert, (size_t) rc))
+ {
+ siridb_insert_free(insert); /* signal is raised */
+ }
+ break;
+ }
+ }
+}
+
+static void on_ping(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ sirinet_pkg_t * package;
+ package = sirinet_pkg_new(pkg->pid, 0, CPROTO_RES_ACK, NULL);
+
+ if (package != NULL)
+ {
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+}
+
+/*
+ * This function can raise a SIGNAL.
+ */
+static void on_reqfile(
+ sirinet_stream_t * client,
+ sirinet_pkg_t * pkg,
+ sirinet_clserver_getfile getfile)
+{
+ CHECK_SIRIDB(client, siridb)
+ siridb_user_t * user = client->origin;
+ sirinet_pkg_t * package = NULL;
+ char err_msg[SIRIDB_MAX_SIZE_ERR_MSG];
+
+ if (siridb->server->flags != SERVER_FLAG_RUNNING)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error getting file because server '%s' having status %d",
+ siridb->server->name,
+ siridb->server->flags);
+ package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_SERVER,
+ err_msg);
+ }
+ else if (!siridb_user_check_access(
+ user,
+ SIRIDB_ACCESS_PROFILE_FULL,
+ err_msg))
+ {
+ package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_USER_ACCESS,
+ err_msg);
+ }
+ else
+ {
+ char * content = NULL;
+ ssize_t size = getfile(&content, siridb);
+ package = (content == NULL) ?
+ sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ CPROTO_ERR_FILE,
+ NULL) :
+ sirinet_pkg_new(
+ pkg->pid,
+ size,
+ CPROTO_RES_FILE,
+ (const unsigned char *) content);
+ free(content);
+ }
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+}
+
+/*
+ * This function can raise a SIGNAL.
+ */
+static void on_register_server(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ CHECK_SIRIDB(client, siridb)
+ siridb_user_t * user = client->origin;
+
+ sirinet_pkg_t * package = NULL;
+ siridb_server_t * new_server = NULL;
+ char err_msg[SIRIDB_MAX_SIZE_ERR_MSG];
+ vec_t * servers = NULL;
+
+ if (siridb->server->flags != SERVER_FLAG_RUNNING)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot register new server because '%s' is having status %d",
+ siridb->server->name,
+ siridb->server->flags);
+ package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_SERVER,
+ err_msg);
+ }
+ if (siridb->flags & SIRIDB_FLAG_REINDEXING)
+ {
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Cannot register new server because the database has not "
+ "finished re-indexing");
+ package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_MSG,
+ err_msg);
+ }
+ else if (!siridb_user_check_access(
+ user,
+ SIRIDB_ACCESS_PROFILE_FULL,
+ err_msg))
+ {
+ package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_USER_ACCESS,
+ err_msg);
+ }
+ else if (!siridb_servers_available(siridb))
+ {
+ sprintf(err_msg, "At least one server is not online or busy");
+ package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_SERVER,
+ err_msg);
+ }
+ else
+ {
+ /* make a copy of the current servers */
+ servers = siridb_servers_other2vec(siridb);
+ if (servers == NULL)
+ {
+ sprintf(err_msg, "Memory allocation error.");
+ package = sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_SERVER,
+ err_msg);
+ }
+ else
+ {
+ log_info("Register a new server");
+ new_server = siridb_server_register(
+ siridb,
+ (unsigned char *) pkg->data,
+ pkg->len);
+
+ pkg->tp = BPROTO_REGISTER_SERVER;
+
+ if (new_server == NULL)
+ {
+ /* a signal might be raised */
+ package = sirinet_pkg_new(pkg->pid, 0, CPROTO_ERR, NULL);
+ }
+ }
+ }
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(client, package);
+ }
+ else if (new_server != NULL)
+ {
+ siridb_server_async_t * server_reg = malloc(
+ sizeof(siridb_server_async_t));
+
+ if (server_reg == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ /* save the client PID so we can respond to the client */
+ server_reg->pid = pkg->pid;
+ server_reg->client = client;
+
+ pkg->tp = BPROTO_REGISTER_SERVER;
+
+ if (servers != NULL && (package = sirinet_pkg_dup(pkg)) != NULL)
+ {
+ /* make sure to decrement the client in the callback */
+ sirinet_stream_incref(client);
+
+ siridb_servers_send_pkg(
+ servers,
+ package,
+ 0,
+ (sirinet_promises_cb)
+ CLSERVER_on_register_server_response,
+ server_reg);
+ }
+ }
+ }
+
+ /* free the servers or NULL */
+ vec_free(servers);
+}
+
+static void on_req_service(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ qp_unpacker_t unpacker;
+ qp_packer_t * packer = NULL;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_obj_t qp_username;
+ qp_obj_t qp_password;
+ qp_obj_t qp_request;
+ sirinet_pkg_t * package = NULL;
+ cproto_server_t tp;
+ char err_msg[SIRI_MAX_SIZE_ERR_MSG];
+
+ if ( qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_username) == QP_RAW &&
+ qp_next(&unpacker, &qp_password) == QP_RAW &&
+ qp_next(&unpacker, &qp_request) == QP_INT64)
+ {
+ tp = (siri_service_account_check(
+ &siri,
+ &qp_username,
+ &qp_password,
+ err_msg)) ?
+ CPROTO_ERR_SERVICE
+ :
+ siri_service_request(
+ qp_request.via.int64,
+ &unpacker,
+ &packer,
+ pkg->pid,
+ client,
+ err_msg);
+
+ package =
+ (tp == CPROTO_DEFERRED) ? NULL :
+ (tp == CPROTO_ERR_SERVICE) ? sirinet_pkg_err(
+ pkg->pid,
+ strlen(err_msg),
+ tp,
+ err_msg) :
+ (tp == CPROTO_ACK_SERVICE_DATA) ? sirinet_packer2pkg(
+ packer,
+ pkg->pid,
+ tp) : sirinet_pkg_new(pkg->pid, 0, tp, NULL);
+ }
+ else
+ {
+ log_error("Invalid service request received.");
+ package = sirinet_pkg_new(
+ pkg->pid,
+ 0,
+ CPROTO_ERR_SERVICE_INVALID_REQUEST,
+ NULL);
+ }
+ if (package != NULL)
+ {
+ switch (package->tp)
+ {
+ case CPROTO_ERR_SERVICE_INVALID_REQUEST:
+ log_warning("Received an invalid manage request.");
+ break;
+ case CPROTO_ERR_SERVICE:
+ log_warning("Error handling manage request: %s", err_msg);
+ break;
+ }
+
+ /* ignore result code, signal can be raised */
+ sirinet_pkg_send(client, package);
+ }
+}
+
+/*
+ * Typedef: sirinet_promises_cb
+ */
+static void CLSERVER_on_register_server_response(
+ vec_t * promises,
+ siridb_server_async_t * server_reg)
+{
+ if (promises != NULL)
+ {
+ sirinet_pkg_t * pkg;
+ sirinet_promise_t * promise;
+ size_t err_count = 0;
+ char err_msg[SIRIDB_MAX_SIZE_ERR_MSG];
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+ if (promise == NULL)
+ {
+ err_count++;
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error occurred while registering the new server on "
+ "at least one server");
+ }
+ else
+ {
+ pkg = promise->data;
+
+ if (pkg == NULL || pkg->tp != BPROTO_ACK_REGISTER_SERVER)
+ {
+ err_count++;
+ snprintf(err_msg,
+ SIRIDB_MAX_SIZE_ERR_MSG,
+ "Error occurred while registering the new server "
+ "on at least '%s'", promise->server->name);
+ }
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+ }
+
+ sirinet_pkg_t * package = (err_count) ?
+ sirinet_pkg_err(
+ server_reg->pid,
+ strlen(err_msg),
+ CPROTO_ERR_MSG,
+ err_msg) :
+ sirinet_pkg_new(
+ server_reg->pid,
+ 0,
+ CPROTO_RES_ACK,
+ NULL);
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(server_reg->client, package);
+ }
+ }
+
+ /* decref the client */
+ sirinet_stream_decref(server_reg->client);
+
+ /* free server register object */
+ free(server_reg);
+}
--- /dev/null
+/*
+ * pipe.c - Named Pipe support.
+ */
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include <siri/net/pipe.h>
+#include <siri/siri.h>
+#include <xpath/xpath.h>
+#include <logger/logger.h>
+
+#define PIPE_NAME_BUF_SZ XPATH_MAX
+
+/*
+ * Return a name for the connection if successful or NULL in case of a failure.
+ *
+ * The returned value is malloced and should be freed.
+ */
+char * sirinet_pipe_name(uv_pipe_t * client)
+{
+ size_t len = PIPE_NAME_BUF_SZ - 1;
+ char * buffer = malloc(PIPE_NAME_BUF_SZ);
+
+ if (buffer == NULL || uv_pipe_getsockname(client, buffer, &len))
+ {
+ free(buffer);
+ return NULL;
+ }
+
+ buffer[len] = '\0';
+ return buffer;
+}
+
+/*
+ * Cleanup socket (pipe) file. (UNUSED)
+ */
+void sirinet_pipe_unlink(uv_pipe_t * client)
+{
+ char * pipe_name = sirinet_pipe_name(client);
+ if (pipe_name != NULL)
+ {
+ log_debug("Unlink named pipe: '%s'", pipe_name);
+ uv_fs_t * req = malloc(sizeof(uv_fs_t));
+ if (req != NULL)
+ {
+ uv_fs_unlink(siri.loop, req, pipe_name, (uv_fs_cb) free);
+ }
+ free(pipe_name);
+ }
+}
--- /dev/null
+/*
+ * pkg.c - SiriDB Package type.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/err.h>
+#include <siri/api.h>
+#include <siri/net/pkg.h>
+#include <siri/net/clserver.h>
+#include <siri/net/protocol.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct pkg_send_s
+{
+ sirinet_pkg_t * pkg;
+ sirinet_stream_t * client;
+} pkg_send_t;
+
+static void PKG_write_cb(uv_write_t * req, int status);
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ * (do not forget to run free(...) on the result. )
+ */
+sirinet_pkg_t * sirinet_pkg_new(
+ uint16_t pid,
+ uint32_t len,
+ uint8_t tp,
+ const unsigned char * data)
+{
+ sirinet_pkg_t * pkg = malloc(sizeof(sirinet_pkg_t) + len);
+
+ if (pkg == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+
+ pkg->len = len;
+ pkg->pid = pid;
+ pkg->tp = tp;
+ pkg->checkbit = 0; /* check bit will be set when send */
+
+ if (data != NULL)
+ {
+ memcpy(pkg->data, data, len);
+ }
+ }
+ return pkg;
+}
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ *
+ * Use 'qp_packer_free' to destroy the returned value or use
+ * 'sirinet_packer2pkg' to convert to 'sirinet_pkg_t'.
+ */
+qp_packer_t * sirinet_packer_new(size_t alloc_size)
+{
+ assert (alloc_size >= sizeof(sirinet_pkg_t));
+
+ qp_packer_t * packer = qp_packer_new(alloc_size);
+ if (packer == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+ packer->len = sizeof(sirinet_pkg_t);
+ }
+
+ return packer;
+}
+
+/*
+ * Returns a 'sirinet_pkg_t' from a packer created with 'sirinet_packer_new'
+ *
+ * Call 'free' to destroy the returned pkg and do not destroy the
+ * packer anymore since this is handled here.
+ */
+sirinet_pkg_t * sirinet_packer2pkg(
+ qp_packer_t * packer,
+ uint16_t pid,
+ uint8_t tp)
+{
+ sirinet_pkg_t * pkg = (sirinet_pkg_t *) packer->buffer;
+
+ pkg->pid = pid;
+ pkg->tp = tp;
+ pkg->len = packer->len - sizeof(sirinet_pkg_t);
+ pkg->checkbit = 0; /* check bit will be set when send */
+
+ /* Free the packer, not the buffer */
+ free(packer);
+
+ return pkg;
+}
+
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ * (do not forget to run free(...) on the result. )
+ */
+sirinet_pkg_t * sirinet_pkg_err(
+ uint16_t pid,
+ uint32_t len,
+ uint8_t tp,
+ const char * msg)
+{
+ assert (msg != NULL);
+
+ sirinet_pkg_t * pkg;
+ qp_packer_t * packer = sirinet_packer_new(len + 20 + sizeof(sirinet_pkg_t));
+ if (packer == NULL)
+ {
+ pkg = NULL; /* signal is raised */
+ }
+ else
+ {
+ qp_add_type(packer, QP_MAP_OPEN);
+ qp_add_raw(packer, (const unsigned char *) "error_msg", 9);
+ qp_add_raw(packer, (const unsigned char *) msg, len);
+ pkg = sirinet_packer2pkg(packer, pid, tp);
+ }
+ return pkg;
+}
+
+static siri_api_header_t pkg__tp_as_ht(uint8_t tp)
+{
+ switch (tp)
+ {
+ /* success */
+ case CPROTO_RES_QUERY:
+ case CPROTO_RES_INSERT:
+ case CPROTO_RES_AUTH_SUCCESS:
+ case CPROTO_RES_ACK:
+ case CPROTO_RES_FILE:
+ case CPROTO_ACK_SERVICE:
+ case CPROTO_ACK_SERVICE_DATA:
+ return E200_OK;
+
+ case CPROTO_ERR_QUERY:
+ case CPROTO_ERR_INSERT:
+ case CPROTO_ERR_SERVICE:
+ case CPROTO_ERR_SERVICE_INVALID_REQUEST:
+ return E400_BAD_REQUEST;
+
+ case CPROTO_ERR_SERVER:
+ case CPROTO_ERR_POOL:
+ return E503_SERVICE_UNAVAILABLE;
+
+ case CPROTO_ERR_USER_ACCESS:
+ case CPROTO_ERR_NOT_AUTHENTICATED:
+ return E403_FORBIDDEN;
+
+ case CPROTO_ERR_AUTH_UNKNOWN_DB:
+ case CPROTO_ERR_AUTH_CREDENTIALS:
+ return E401_UNAUTHORIZED;
+ }
+ return E500_INTERNAL_SERVER_ERROR;
+}
+
+/*
+ * Returns 0 if successful or -1 when an error has occurred.
+ * (signal is raised in case of an error)
+ *
+ * Note: pkg will be freed after calling this function.
+ */
+int sirinet_pkg_send(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ if (client->tp == STREAM_API_CLIENT)
+ {
+ siri_api_send(
+ (siri_api_request_t *) client,
+ pkg__tp_as_ht(pkg->tp),
+ pkg->data,
+ pkg->len);
+ free(pkg);
+ return 0;
+ }
+
+ uv_write_t * req = malloc(sizeof(uv_write_t));
+
+ if (req == NULL)
+ {
+ ERR_ALLOC
+ free(pkg);
+ return -1;
+ }
+
+ pkg_send_t * data = malloc(sizeof(pkg_send_t));
+
+ if (data == NULL)
+ {
+ ERR_ALLOC
+ free(pkg);
+ free(req);
+ return -1;
+ }
+
+ /* increment client reference counter */
+ sirinet_stream_incref(client);
+
+ data->client = client;
+ data->pkg = pkg;
+ req->data = data;
+
+ /* set the correct check bit */
+ pkg->checkbit = pkg->tp ^ 255;
+
+ uv_buf_t wrbuf = uv_buf_init(
+ (char *) pkg,
+ sizeof(sirinet_pkg_t) + pkg->len);
+
+ if (uv_write(req, client->stream, &wrbuf, 1, PKG_write_cb))
+ {
+ sirinet_stream_decref(data->client);
+ free(pkg);
+ free(req);
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns a copy of package allocated using malloc().
+ * In case of an error, NULL is returned.
+ */
+sirinet_pkg_t * sirinet_pkg_dup(sirinet_pkg_t * pkg)
+{
+ size_t size = sizeof(sirinet_pkg_t) + pkg->len;
+ sirinet_pkg_t * dup = malloc(size);
+ if (dup != NULL)
+ {
+ memcpy(dup, pkg, size);
+ }
+ return dup;
+}
+
+static void PKG_write_cb(uv_write_t * req, int status)
+{
+ if (status)
+ {
+ log_error("Socket write error: %s", uv_strerror(status));
+ }
+
+ pkg_send_t * data = (pkg_send_t *) req->data;
+
+ sirinet_stream_decref(data->client);
+
+ free(data->pkg);
+ free(data);
+ free(req);
+}
--- /dev/null
+/*
+ * promise.c - Promise used for sending to data to SiriDB servers.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/err.h>
+#include <siri/net/promise.h>
+
+const char * sirinet_promise_strstatus(sirinet_promise_status_t status)
+{
+ switch (status)
+ {
+ case PROMISE_SUCCESS: return "success";
+ case PROMISE_TIMEOUT_ERROR: return "timed out";
+ case PROMISE_WRITE_ERROR: return "write error";
+ case PROMISE_CANCELLED_ERROR: return "cancelled";
+ case PROMISE_PKG_TYPE_ERROR: return "unexpected package type";
+ }
+ /* all cases MUST be handled */
+ assert(0);
+ return "";
+}
+
+/*
+ * Creating a new promise is done in 'siridb_server_send_pkg'.
+ */
+
+
--- /dev/null
+/*
+ * promises.c - Collection for promised.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/err.h>
+#include <siri/net/promise.h>
+#include <siri/net/promises.h>
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ */
+sirinet_promises_t * sirinet_promises_new(
+ size_t size,
+ sirinet_promises_cb cb,
+ void * data,
+ sirinet_pkg_t * pkg)
+{
+ sirinet_promises_t * promises = malloc(sizeof(sirinet_promises_t));
+ if (promises == NULL)
+ {
+ ERR_ALLOC
+ }
+ else
+ {
+
+ promises->cb = cb;
+ promises->data = data;
+ promises->promises = vec_new(size);
+ promises->pkg = pkg;
+
+ if (promises->promises == NULL)
+ {
+ free(promises);
+ promises = NULL;
+ ERR_ALLOC
+ }
+ }
+ return promises;
+}
+
+/*
+ * This function can be used to free promises with data. It assumes
+ * data simple can be destroyed with simple calling free().
+ */
+void sirinet_promises_llist_free(vec_t * promises)
+{
+ sirinet_promise_t * promise;
+ size_t i;
+
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = (sirinet_promise_t *) promises->data[i];
+ if (promise != NULL)
+ {
+ /* make sure we free the promise and data */
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+ }
+}
+
+/*
+ * This function will clean the promises type and list. The promises->cb
+ * is responsible for calling 'sirinet_promise_decref' on each promise and
+ * free promise->data.
+ *
+ * A promise reference count will be incremented by one.
+ */
+void sirinet_promises_on_response(
+ sirinet_promise_t * promise,
+ sirinet_pkg_t * pkg,
+ int status)
+{
+ sirinet_promises_t * promises = (sirinet_promises_t *) promise->data;
+
+ if (status)
+ {
+ /* we already have a log entry so this can be a debug log */
+ log_debug(
+ "Error occurred while sending package to '%s' (%s)",
+ promise->server->name,
+ sirinet_promise_strstatus((sirinet_promise_status_t) status));
+ promise->data = NULL;
+ }
+ else
+ {
+ /* we can ignore errors from sirinet_pkg_dup() */
+ promise->data = (void *) sirinet_pkg_dup(pkg);
+ }
+
+ vec_append(promises->promises, (void *) promise);
+
+ SIRINET_PROMISES_CHECK(promises)
+}
--- /dev/null
+/*
+ * protocol.c - Protocol definitions for SiriDB.
+ */
+#include <siri/net/protocol.h>
+#include <stdio.h>
+
+char protocol_str[512];
+
+const char * sirinet_cproto_client_str(cproto_client_t n)
+{
+ switch (n)
+ {
+ case CPROTO_REQ_QUERY: return "CPROTO_REQ_QUERY";
+ case CPROTO_REQ_INSERT: return "CPROTO_REQ_INSERT";
+ case CPROTO_REQ_AUTH: return "CPROTO_REQ_AUTH";
+ case CPROTO_REQ_PING: return "CPROTO_REQ_PING";
+
+ /* start internal usage */
+ case CPROTO_REQ_REGISTER_SERVER: return "CPROTO_REQ_REGISTER_SERVER";
+ case CPROTO_REQ_FILE_SERVERS: return "CPROTO_REQ_FILE_SERVERS";
+ case CPROTO_REQ_FILE_USERS: return "CPROTO_REQ_FILE_USERS";
+ case CPROTO_REQ_FILE_GROUPS: return "CPROTO_REQ_FILE_GROUPS";
+ case CPROTO_REQ_FILE_DATABASE: return "CPROTO_REQ_FILE_DATABASE";
+ /* end internal usage */
+
+ case CPROTO_REQ_SERVICE: return "CPROTO_REQ_SERVICE";
+
+ default:
+ sprintf(protocol_str, "CPROTO_CLIENT_TYPE_UNKNOWN (%d)", n);
+ return protocol_str;
+ }
+}
+
+const char * sirinet_cproto_server_str(cproto_server_t n)
+{
+ switch (n)
+ {
+ case CPROTO_RES_QUERY: return "CPROTO_RES_QUERY";
+ case CPROTO_RES_INSERT: return "CPROTO_RES_INSERT";
+ case CPROTO_RES_AUTH_SUCCESS: return "CPROTO_RES_AUTH_SUCCESS";
+ case CPROTO_RES_ACK: return "CPROTO_RES_ACK";
+ case CPROTO_RES_FILE: return "CPROTO_RES_FILE";
+
+ case CPROTO_ACK_SERVICE: return "CPROTO_ACK_SERVICE";
+ case CPROTO_ACK_SERVICE_DATA: return "CPROTO_ACK_SERVICE_DATA";
+
+ case CPROTO_ERR_MSG: return "CPROTO_ERR_MSG";
+ case CPROTO_ERR_QUERY: return "CPROTO_ERR_QUERY";
+ case CPROTO_ERR_INSERT: return "CPROTO_ERR_INSERT";
+ case CPROTO_ERR_SERVER: return "CPROTO_ERR_SERVER";
+ case CPROTO_ERR_POOL: return "CPROTO_ERR_POOL";
+ case CPROTO_ERR_USER_ACCESS: return "CPROTO_ERR_USER_ACCESS";
+ case CPROTO_ERR: return "CPROTO_ERR";
+ case CPROTO_ERR_NOT_AUTHENTICATED: return "CPROTO_ERR_NOT_AUTHENTICATED";
+ case CPROTO_ERR_AUTH_CREDENTIALS: return "CPROTO_ERR_AUTH_CREDENTIALS";
+ case CPROTO_ERR_AUTH_UNKNOWN_DB: return "CPROTO_ERR_AUTH_UNKNOWN_DB";
+ case CPROTO_ERR_FILE: return "CPROTO_ERR_FILE";
+ case CPROTO_ERR_SERVICE: return "CPROTO_ERR_SERVICE";
+ case CPROTO_ERR_SERVICE_INVALID_REQUEST: return "CPROTO_ERR_SERVICE_INVALID_REQUEST";
+ default:
+ sprintf(protocol_str, "CPROTO_SERVER_TYPE_UNKNOWN (%d)", n);
+ return protocol_str;
+ }
+}
+
+const char * sirinet_bproto_client_str(bproto_client_t n)
+{
+ switch (n)
+ {
+ case BPROTO_AUTH_REQUEST: return "BPROTO_AUTH_REQUEST";
+ case BPROTO_FLAGS_UPDATE: return "BPROTO_FLAGS_UPDATE";
+ case BPROTO_LOG_LEVEL_UPDATE: return "BPROTO_LOG_LEVEL_UPDATE";
+ case BPROTO_REPL_FINISHED: return "BPROTO_REPL_FINISHED";
+ case BPROTO_QUERY_SERVER: return "BPROTO_QUERY_SERVER";
+ case BPROTO_QUERY_UPDATE: return "BPROTO_QUERY_UPDATE";
+ case BPROTO_INSERT_POOL: return "BPROTO_INSERT_POOL";
+ case BPROTO_INSERT_SERVER: return "BPROTO_INSERT_SERVER";
+ case BPROTO_INSERT_TEST_POOL: return "BPROTO_INSERT_TEST_POOL";
+ case BPROTO_INSERT_TEST_SERVER: return "BPROTO_INSERT_TEST_SERVER";
+ case BPROTO_INSERT_TESTED_POOL: return "BPROTO_INSERT_TESTED_POOL";
+ case BPROTO_INSERT_TESTED_SERVER: return "BPROTO_INSERT_TESTED_SERVER";
+ case BPROTO_REGISTER_SERVER: return "BPROTO_REGISTER_SERVER";
+ case BPROTO_DROP_SERIES: return "BPROTO_DROP_SERIES";
+ case BPROTO_REQ_GROUPS: return "BPROTO_REQ_GROUPS";
+ case BPROTO_ENABLE_BACKUP_MODE: return "BPROTO_ENABLE_BACKUP_MODE";
+ case BPROTO_DISABLE_BACKUP_MODE: return "BPROTO_DISABLE_BACKUP_MODE";
+ case BPROTO_TEE_PIPE_NAME_UPDATE: return "BPROTO_TEE_PIPE_NAME_UPDATE";
+ case BPROTO_DROP_DATABASE: return "BPROTO_DROP_DATABASE";
+ case BPROTO_REQ_TAGS: return "BPROTO_REQ_TAGS";
+ case BPROTO_SERIES_TAGS: return "BPROTO_SERIES_TAGS";
+ case BPROTO_EMPTY_TAGS: return "BPROTO_EMPTY_TAGS";
+ default:
+ sprintf(protocol_str, "BPROTO_CLIENT_TYPE_UNKNOWN (%d)", n);
+ return protocol_str;
+ }
+}
+
+const char * sirinet_bproto_server_str(bproto_server_t n)
+{
+ switch (n)
+ {
+ case BPROTO_RES_QUERY: return "BPROTO_RES_QUERY";
+ case BPROTO_ERR_QUERY: return "BPROTO_ERR_QUERY";
+ case BPROTO_ERR_SERVER: return "BPROTO_ERR_SERVER";
+ case BPROTO_ERR_POOL: return "BPROTO_ERR_POOL";
+ case BPROTO_AUTH_ERR_UNKNOWN_UUID: return "BPROTO_AUTH_ERR_UNKNOWN_UUID";
+ case BPROTO_AUTH_ERR_UNKNOWN_DBNAME: return "BPROTO_AUTH_ERR_UNKNOWN_DBNAME";
+ case BPROTO_AUTH_ERR_INVALID_UUID: return "BPROTO_AUTH_ERR_INVALID_UUID";
+ case BPROTO_AUTH_ERR_VERSION_TOO_OLD: return "BPROTO_AUTH_ERR_VERSION_TOO_OLD";
+ case BPROTO_AUTH_ERR_VERSION_TOO_NEW: return "BPROTO_AUTH_ERR_VERSION_TOO_NEW";
+ case BPROTO_ERR_NOT_AUTHENTICATED: return "BPROTO_ERR_NOT_AUTHENTICATED";
+ case BPROTO_ERR_INSERT: return "BPROTO_ERR_INSERT";
+ case BPROTO_ERR_REGISTER_SERVER: return "BPROTO_ERR_REGISTER_SERVER";
+ case BPROTO_ERR_DROP_SERIES: return "BPROTO_ERR_DROP_SERIES";
+ case BPROTO_ERR_ENABLE_BACKUP_MODE: return "BPROTO_ERR_ENABLE_BACKUP_MODE";
+ case BPROTO_ERR_DISABLE_BACKUP_MODE: return "BPROTO_ERR_DISABLE_BACKUP_MODE";
+ case BPROTO_AUTH_SUCCESS: return "BPROTO_AUTH_SUCCESS";
+ case BPROTO_ACK_FLAGS: return "BPROTO_ACK_FLAGS";
+ case BPROTO_ACK_LOG_LEVEL: return "BPROTO_ACK_LOG_LEVEL";
+ case BPROTO_ACK_INSERT: return "BPROTO_ACK_INSERT";
+ case BPROTO_ACK_REPL_FINISHED: return "BPROTO_ACK_REPL_FINISHED";
+ case BPROTO_ACK_REGISTER_SERVER: return "BPROTO_ACK_REGISTER_SERVER";
+ case BPROTO_ACK_DROP_SERIES: return "BPROTO_ACK_DROP_SERIES";
+ case BPROTO_ACK_ENABLE_BACKUP_MODE: return "BPROTO_ACK_ENABLE_BACKUP_MODE";
+ case BPROTO_ACK_DISABLE_BACKUP_MODE: return "BPROTO_ACK_DISABLE_BACKUP_MODE";
+ case BPROTO_RES_GROUPS: return "BPROTO_RES_GROUPS";
+ case BPROTO_ACK_TEE_PIPE_NAME: return "BPROTO_ACK_TEE_PIPE_NAME";
+ case BPROTO_ACK_DROP_DATABASE: return "BPROTO_ACK_DROP_DATABASE";
+ case BPROTO_RES_TAGS: return "BPROTO_RES_TAGS";
+ case BPROTO_ACK_SERIES_TAGS: return "BPROTO_ACK_SERIES_TAGS";
+ case BPROTO_ACK_EMPTY_TAGS: return "BPROTO_ACK_EMPTY_TAGS";
+ default:
+ sprintf(protocol_str, "BPROTO_SERVER_TYPE_UNKNOWN (%d)", n);
+ return protocol_str;
+ }
+}
--- /dev/null
+/*
+ * stream.c - For handling streams.
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/service/client.h>
+#include <siri/err.h>
+#include <siri/net/protocol.h>
+#include <siri/net/stream.h>
+#include <siri/net/pipe.h>
+#include <siri/net/tcp.h>
+#include <siri/siri.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define MAX_ALLOWED_PKG_SIZE 41943040 /* 40 MB */
+
+#define QUIT_STREAM \
+ free(client->buf); \
+ client->buf = NULL; \
+ client->len = 0; \
+ client->size = 0; \
+ client->on_data = NULL; \
+ sirinet_stream_decref(client); \
+ return;
+
+/*
+ * Returns NULL and raises a SIGNAL in case an error has occurred.
+ *
+ * Note: object->ref is initially set to 1
+ */
+sirinet_stream_t * sirinet_stream_new(sirinet_stream_tp_t tp, on_data_cb_t cb)
+{
+ size_t stream_sz;
+ sirinet_stream_t * client = malloc(sizeof(sirinet_stream_t));
+
+ if (client == NULL)
+ {
+ ERR_ALLOC
+ return NULL;
+ }
+
+ client->tp = tp;
+ client->on_data = cb;
+ client->buf = NULL;
+ client->len = 0;
+ client->size = -1; /* this will force allocating on first request */
+ client->origin = NULL;
+ client->siridb = NULL;
+ client->ref = 1;
+
+ switch(tp)
+ {
+ case STREAM_API_CLIENT:
+ case STREAM_TCP_CLIENT:
+ case STREAM_TCP_BACKEND:
+ case STREAM_TCP_SERVER:
+ case STREAM_TCP_MANAGE:
+ stream_sz = sizeof(uv_tcp_t);
+ break;
+ case STREAM_PIPE_CLIENT:
+ stream_sz = sizeof(uv_pipe_t);
+ break;
+ default:
+ stream_sz = sizeof(uv_stream_t);
+ assert(0);
+ }
+
+ client->stream = malloc(stream_sz);
+ if (client->stream == NULL)
+ {
+ free(client);
+ ERR_ALLOC
+ return NULL;
+ }
+
+ client->stream->data = client;
+
+ return client;
+}
+
+/*
+ * Return a name for the connection if successful or NULL in case of a failure.
+ *
+ * The returned value is malloced and should be freed.
+ */
+char * sirinet_stream_name(sirinet_stream_t * client)
+{
+ switch ((sirinet_stream_tp_t) client->tp)
+ {
+ case STREAM_API_CLIENT:
+ case STREAM_TCP_CLIENT:
+ case STREAM_TCP_BACKEND:
+ case STREAM_TCP_SERVER:
+ case STREAM_TCP_MANAGE:
+ return sirinet_tcp_name((uv_tcp_t *) client->stream);
+ case STREAM_PIPE_CLIENT:
+ return sirinet_pipe_name((uv_pipe_t *) client->stream);
+ }
+ return NULL;
+}
+
+/*
+ * This function can raise a SIGNAL.
+ */
+void sirinet_stream_alloc_buffer(
+ uv_handle_t * handle,
+ size_t suggested_size,
+ uv_buf_t * buf)
+{
+ sirinet_stream_t * client = (sirinet_stream_t *) handle->data;
+
+ if (!client->len && client->size > RESET_BUF_SIZE)
+ {
+ free(client->buf);
+ client->buf = malloc(suggested_size);
+ if (client->buf == NULL)
+ {
+ ERR_ALLOC
+ buf->len = 0;
+ return;
+ }
+ client->size = suggested_size;
+ client->len = 0;
+ }
+ buf->base = client->buf + client->len;
+ buf->len = client->size - client->len;
+}
+
+/*
+ * This function can raise a SIGNAL.
+ */
+void sirinet_stream_on_data(
+ uv_stream_t * uvclient,
+ ssize_t nread,
+ const uv_buf_t * buf)
+{
+ sirinet_stream_t * client = uvclient->data;
+ sirinet_pkg_t * pkg;
+ size_t total_sz;
+ uint8_t check;
+
+ /* this functions should not be used for the API client */
+ assert (client->tp != STREAM_API_CLIENT);
+
+ /*
+ * client->on_data is NULL when 'sirinet_stream_decref' is called from
+ * within this function. We should never call 'sirinet_stream_decref' twice
+ * so the best thing is to log and and exit this function.
+ */
+ if (client->on_data == NULL)
+ {
+ char * name = sirinet_stream_name(client);
+ if (name != NULL)
+ {
+ log_error(
+ "Received data from '%s' but we ignore the data since the "
+ "connection will be closed in a few seconds...",
+ name);
+ free(name);
+ }
+ return;
+ }
+
+ if (nread < 0)
+ {
+ if (nread != UV_EOF)
+ {
+ log_error("Read error: %s", uv_err_name(nread));
+ }
+ QUIT_STREAM
+ }
+
+ client->len += nread;
+
+ if (client->len < sizeof(sirinet_pkg_t))
+ {
+ return;
+ }
+
+ pkg = (sirinet_pkg_t *) client->buf;
+ check = pkg->tp ^ 255;
+ if (check != pkg->checkbit ||
+ (( client->tp == STREAM_TCP_CLIENT ||
+ client->tp == STREAM_PIPE_CLIENT) &&
+ pkg->len > MAX_ALLOWED_PKG_SIZE))
+ {
+ char * name = sirinet_stream_name(client);
+ if (name != NULL)
+ {
+ log_error(
+ "Got an illegal package or size too large from '%s', "
+ "closing connection "
+ "(pid: %" PRIu16 ", len: %" PRIu32 ", tp: %" PRIu8 ")",
+ name, pkg->pid, pkg->len, pkg->tp);
+ free(name);
+ }
+ QUIT_STREAM
+ }
+
+ total_sz = sizeof(sirinet_pkg_t) + pkg->len;
+ if (client->len < total_sz)
+ {
+ if (client->size < total_sz)
+ {
+ char * tmp = realloc(client->buf, total_sz);
+ if (tmp == NULL)
+ {
+ log_critical(
+ "Cannot allocate size for package "
+ "(pid: %" PRIu16 ", len: %" PRIu32 ", tp: %" PRIu8 ")",
+ pkg->pid, pkg->len, pkg->tp);
+ QUIT_STREAM
+ }
+ client->buf = tmp;
+ client->size = total_sz;
+ }
+ return;
+ }
+
+ /* call on-data function */
+ (*client->on_data)(client, pkg);
+
+ client->len -= total_sz;
+
+ if (client->len > 0)
+ {
+ /* move data and call sirinet_stream_on_data() function again */
+ memmove(client->buf, client->buf + total_sz, client->len);
+ sirinet_stream_on_data(uvclient, 0, buf);
+ }
+}
+
+/*
+ * Never use this function but call sirinet_stream_decref.
+ * Destroy stream. (parsing NULL is not allowed)
+ *
+ * We know three different stream types:
+ * - client: used for clients. a user object might be destroyed.
+ * - back-end: used to connect to other servers. a server might be destroyed.
+ * - server: user for severs connecting to here. a server might be destroyed.
+ *
+ * In case a server is destroyed, remaining promises will be cancelled and
+ * the call-back functions will be called.
+ */
+void sirinet__stream_free(uv_stream_t * uvclient)
+{
+ sirinet_stream_t * client = uvclient->data;
+
+ switch ((sirinet_stream_tp_t) client->tp)
+ {
+ case STREAM_API_CLIENT:
+ case STREAM_PIPE_CLIENT:
+ case STREAM_TCP_CLIENT: /* listens to client connections */
+ log_debug("client connection lost");
+ if (client->origin != NULL)
+ {
+ siridb_user_t * user = client->origin;
+ siridb_user_decref(user);
+ }
+ break;
+ case STREAM_TCP_BACKEND: /* listens to server connections */
+ if (client->origin != NULL)
+ {
+ siridb_server_t * server = (siridb_server_t *) client->origin;
+ siridb_server_decref(server);
+ }
+ break;
+ case STREAM_TCP_SERVER: /* a server connection */
+ {
+ siridb_server_t * server = client->origin;
+ server->client = NULL;
+ server->flags = 0;
+ siridb_server_decref(server);
+ }
+ break;
+ case STREAM_TCP_MANAGE: /* a server manage connection */
+ siri_service_client_free((siri_service_client_t *) client->origin);
+ siri.client = NULL;
+ assert (client->siridb == NULL);
+ break;
+ }
+ if (client->siridb)
+ {
+ siridb_decref(client->siridb);
+ }
+ free(client->buf);
+ free(client);
+ free(uvclient);
+}
+
+
+
+
+
+
--- /dev/null
+/*
+ * tcp.c - TCP support.
+ */
+#include <siri/net/tcp.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <uv.h>
+#include <xstr/xstr.h>
+#include <siri/cfg/cfg.h>
+#include <logger/logger.h>
+
+#define TCP_NAME_BUF_SZ 54
+
+const char * sirinet_tcp_ip_support_str(uint8_t ip_support)
+{
+ switch (ip_support)
+ {
+ case IP_SUPPORT_ALL: return "ALL";
+ case IP_SUPPORT_IPV4ONLY: return "IPV4ONLY";
+ case IP_SUPPORT_IPV6ONLY: return "IPV6ONLY";
+ default: return "UNKNOWN";
+ }
+}
+
+/*
+ * Return a name for the connection if successful or NULL in case of a failure.
+ *
+ * The returned value is malloced and should be freed.
+ */
+char * sirinet_tcp_name(uv_tcp_t * client)
+{
+ char * buffer = malloc(TCP_NAME_BUF_SZ);
+ struct sockaddr_storage name;
+ int namelen = sizeof(name);
+
+ if (buffer == NULL ||
+ uv_tcp_getpeername(client, (struct sockaddr *) &name, &namelen))
+ {
+ goto failed;
+ }
+
+ switch (name.ss_family)
+ {
+ case AF_INET:
+ {
+ char addr[INET_ADDRSTRLEN];
+ uv_inet_ntop(
+ AF_INET,
+ &((struct sockaddr_in *) &name)->sin_addr,
+ addr,
+ sizeof(addr));
+ sprintf(
+ buffer,
+ "%s:%d",
+ addr,
+ ntohs(((struct sockaddr_in *) &name)->sin_port));
+ }
+ break;
+
+ case AF_INET6:
+ {
+ char addr[INET6_ADDRSTRLEN];
+ uv_inet_ntop(
+ AF_INET6,
+ &((struct sockaddr_in6 *) &name)->sin6_addr,
+ addr,
+ sizeof(addr));
+ sprintf(
+ buffer,
+ "[%s]:%d",
+ addr,
+ ntohs(((struct sockaddr_in6 *) &name)->sin6_port));
+ }
+ break;
+
+ default:
+ goto failed;
+ }
+
+ return buffer;
+
+failed:
+ free(buffer);
+ return NULL;
+}
+
+int sirinet_extract_addr_port(
+ char * s,
+ char * addr,
+ uint16_t * addrport)
+{
+ int test_port;
+ char * strport;
+ char * address;
+ char hostname[SIRI_CFG_MAX_LEN_ADDRESS];
+ if (gethostname(hostname, SIRI_CFG_MAX_LEN_ADDRESS))
+ {
+ log_debug(
+ "Unable to read the systems host name. Since its only purpose "
+ "is to apply this in the configuration file this might not be "
+ "any problem. (using 'localhost' as fallback)");
+ strcpy(hostname, "localhost");
+ }
+
+
+ if (*s == '[')
+ {
+ /* an IPv6 address... */
+ for (strport = address = s + 1; *strport; strport++)
+ {
+ if (*strport == ']')
+ {
+ *strport = 0;
+ strport++;
+ break;
+ }
+ }
+ }
+ else
+ {
+ strport = address = s;
+ }
+
+ for (; *strport; strport++)
+ {
+ if (*strport == ':')
+ {
+ *strport = 0;
+ strport++;
+ break;
+ }
+ }
+
+ if (!strlen(address) || strlen(address) >= SIRI_CFG_MAX_LEN_ADDRESS)
+ {
+ log_error("error: got an unexpected address value '%s'.", s);
+ return -1;
+ }
+
+ if (strcpy(addr, address) == NULL || xstr_replace_str(
+ addr,
+ "%HOSTNAME",
+ hostname,
+ SIRI_CFG_MAX_LEN_ADDRESS))
+ {
+ log_critical("memory allocation while copying address");
+ return -1;
+ }
+
+ if (xstr_is_int(strport))
+ {
+ test_port = atoi(strport);
+ if (test_port < 1 || test_port > 65535)
+ {
+ log_error(
+ "error: port should be between 1 and 65535, got '%d'.",
+ test_port);
+ return -1;
+ }
+
+ *addrport = (uint16_t) test_port;
+ }
+
+ log_debug("Read '%s': %s:%d",
+ s,
+ addr,
+ *addrport);
+ return 0;
+}
+
+
+
--- /dev/null
+/*
+ * optimize.c - Optimize task SiriDB.
+ *
+ * There is one and only one optimize task thread running for SiriDB. For this
+ * reason we do not need to parse data but we should only take care for locks
+ * while writing data.
+ *
+ * Thread debugging:
+ * log_debug("getpid: %d - pthread_self: %lu",getpid(), pthread_self());
+ *
+ */
+#include <assert.h>
+#include <logger/logger.h>
+#include <siri/db/shard.h>
+#include <siri/db/shards.h>
+#include <siri/optimize.h>
+#include <siri/siri.h>
+#include <vec/vec.h>
+#include <unistd.h>
+
+static siri_optimize_t optimize = {
+ .pause=0,
+ .status=SIRI_OPTIMIZE_PENDING,
+ .idx_fp=NULL,
+ .idx_fn=NULL
+};
+
+static void OPTIMIZE_work(uv_work_t * work);
+static void OPTIMIZE_cleanup(vec_t * slsiridb);
+static void OPTIMIZE_work_finish(uv_work_t * work, int status);
+static void OPTIMIZE_cb(uv_timer_t * handle);
+
+void siri_optimize_init(siri_t * siri)
+{
+ /*
+ * Main Thread
+ */
+
+ uint64_t timeout = siri->cfg->optimize_interval * 1000;
+ siri->optimize = &optimize;
+ uv_timer_init(siri->loop, &optimize.timer);
+
+ /* do not start with optimize_interval zero */
+ if (timeout)
+ {
+ uv_timer_start(&optimize.timer, OPTIMIZE_cb, timeout, timeout);
+ }
+ else
+ {
+ log_warning(
+ "Optimizing is disabled! This is not recommended and "
+ "can be enabled by changing the optimize_interval in the "
+ "configuration file to a positive integer value.");
+ }
+}
+
+void siri_optimize_stop(void)
+{
+ /*
+ * Main Thread
+ */
+
+ /* uv_cancel will only be successful when the task is not started yet */
+ optimize.status = SIRI_OPTIMIZE_CANCELLED;
+ optimize.pause = 0;
+ uv_cancel((uv_req_t *) &optimize.work);
+
+ /* stop the timer so it will not run again */
+ uv_timer_stop(&optimize.timer);
+ uv_close((uv_handle_t *) &optimize.timer, NULL);
+
+ /* keep optimize bound to siri because we still want to check the status */
+}
+
+/*
+ * Increment pause. This is not just a simple boolean because more than one
+ * siridb database can pause the optimize task.
+ */
+void siri_optimize_pause(void)
+{
+ optimize.pause++;
+ if (optimize.status == SIRI_OPTIMIZE_PENDING)
+ {
+ optimize.status = SIRI_OPTIMIZE_PAUSED_MAIN;
+ }
+}
+
+/*
+ * Decrement pause. This is not just a simple boolean because more than one
+ * siridb database can pause the optimize task.
+ */
+void siri_optimize_continue(void)
+{
+ assert (optimize.pause);
+ if (!--optimize.pause && optimize.status == SIRI_OPTIMIZE_PAUSED_MAIN)
+ {
+ log_debug("Optimize task was paused by the main thread, continue...");
+ optimize.status = SIRI_OPTIMIZE_PENDING;
+ }
+}
+
+/*
+ * This function should only be called from the optimize thread and waits
+ * if the optimize task is paused. The optimize status after the pause is
+ * returned.
+ */
+int siri_optimize_wait(void)
+{
+ /* its possible that another database is paused, but we wait anyway */
+ if (optimize.pause)
+ {
+ assert (optimize.status == SIRI_OPTIMIZE_RUNNING);
+ optimize.status = SIRI_OPTIMIZE_PAUSED;
+
+ /* close open index file in case this is required */
+ if (optimize.idx_fp != NULL)
+ {
+ log_info("Closing index file: '%s'", optimize.idx_fn);
+ if (fclose(optimize.idx_fp))
+ {
+ log_critical(
+ "Closing index file failed: '%s'",
+ optimize.idx_fn);
+ }
+ optimize.idx_fp = NULL;
+ }
+
+ log_info("Optimize task is paused, wait until we can continue...");
+
+ sleep(5);
+
+ while (optimize.pause)
+ {
+ log_debug("Optimize task is still paused, wait for 5 seconds...");
+ sleep(5);
+ }
+
+ switch (optimize.status)
+ {
+ case SIRI_OPTIMIZE_PAUSED:
+ log_info("Continue optimize task...");
+ optimize.status = SIRI_OPTIMIZE_RUNNING;
+
+ if (optimize.idx_fn != NULL &&
+ (optimize.idx_fp = fopen(optimize.idx_fn, "a")) == NULL)
+ {
+ log_error("Cannot re-open index file: '%s'", optimize.idx_fn);
+ free(optimize.idx_fn);
+ optimize.idx_fn = NULL;
+ }
+
+ break;
+
+ case SIRI_OPTIMIZE_CANCELLED:
+ log_info("Optimize task is cancelled.");
+ break;
+
+ default:
+ assert (0);
+ break;
+ }
+
+ }
+ return optimize.status;
+}
+
+/*
+ * Create an index file created from the given file name. (The index file
+ * will be equal the the given file name except for the extension which will
+ * be changed to .idx
+ *
+ * Returns 0 if successful and -1 in case of an error. In case of an error
+ * both optimize.idx_fn and optimize.idx_fp will be NULL.
+ */
+int siri_optimize_create_idx(const char * fn)
+{
+ assert (optimize.idx_fn == NULL && strlen(fn) > 3);
+
+ /* copy file name */
+ optimize.idx_fn = strdup(fn);
+ if (optimize.idx_fn == NULL)
+ {
+ log_error("Memory allocation error");
+ return -1;
+ }
+
+ /* replace last three characters from sdb to idx */
+ memcpy(optimize.idx_fn + strlen(fn) - 3, "idx", 3);
+
+ /* open file for writing */
+ optimize.idx_fp = fopen(optimize.idx_fn, "w");
+ if (optimize.idx_fp == NULL)
+ {
+ log_error(
+ "Cannot open index file for writing: '%s'",
+ optimize.idx_fn);
+ free(optimize.idx_fn);
+ optimize.idx_fn = NULL;
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * When a shard optimization has finished, this function closes the temporary
+ * index file and renames the file to the final name. (__ will be stripped
+ * from the file name)
+ *
+ * If no index file was created then this function simply return 0.
+ * Argument 'remove_old' should be only set to true (1) in case the 'old'
+ * shard file had an index which can be removed.
+ */
+int siri_optimize_finish_idx(const char * fn, int remove_old)
+{
+ int rc = 0;
+
+ siridb_shard_idx_file(buffer, fn);
+
+ if (optimize.idx_fn == NULL)
+ {
+ log_warning("No index file was created");
+ return 0;
+ }
+
+ if (fclose(optimize.idx_fp))
+ {
+ log_critical("Closing index file failed: '%s'", optimize.idx_fn);
+ rc = -1;
+ }
+
+ if (remove_old && unlink(buffer))
+ {
+ log_warning("Cannot remove file: '%s'", buffer);
+ }
+
+ optimize.idx_fp = NULL;
+
+ if (rename(optimize.idx_fn, buffer))
+ {
+ log_critical(
+ "Rename failed: '%s' to '%s'",
+ optimize.idx_fn,
+ buffer);
+ rc = -1;
+ }
+
+ free(optimize.idx_fn);
+ optimize.idx_fn = NULL;
+
+ return rc;
+}
+
+static void OPTIMIZE_work(uv_work_t * work __attribute__((unused)))
+{
+ /*
+ * Optimize Thread
+ */
+
+ vec_t * slsiridb;
+ vec_t * slshards;
+ siridb_t * siridb;
+ siridb_shard_t * shard;
+ uint8_t c = siri.cfg->shard_compression;
+ size_t i;
+ uint64_t expi[2];
+
+ log_info("Start optimize task");
+
+ if (siri_optimize_wait() == SIRI_OPTIMIZE_CANCELLED)
+ {
+ return;
+ }
+
+ uv_mutex_lock(&siri.siridb_mutex);
+
+ slsiridb = llist2vec(siri.siridb_list);
+ if (slsiridb != NULL)
+ {
+ for (i = 0; i < slsiridb->len; i++)
+ {
+ siridb = (siridb_t *) slsiridb->data[i];
+ siridb_incref(siridb);
+ }
+ }
+
+ uv_mutex_unlock(&siri.siridb_mutex);
+
+ if (siri_err || slsiridb == NULL)
+ {
+ OPTIMIZE_cleanup(slsiridb);
+ return;
+ }
+
+ for (i = 0; i < slsiridb->len; i++)
+ {
+ size_t j;
+ siridb = (siridb_t *) slsiridb->data[i];
+
+ log_debug("Start optimizing database '%s'", siridb->dbname);
+
+ uv_mutex_lock(&siridb->shards_mutex);
+
+ slshards = siridb_shards_vec(siridb);
+
+ uv_mutex_unlock(&siridb->shards_mutex);
+
+ uv_mutex_lock(&siridb->values_mutex);
+
+ expi[SIRIDB_SHARD_TP_NUMBER] = siridb->exp_at_num;
+ expi[SIRIDB_SHARD_TP_LOG] = siridb->exp_at_log;
+
+ uv_mutex_unlock(&siridb->values_mutex);
+
+ if (slshards == NULL)
+ {
+ log_error("Error creating reference list for shards.");
+ OPTIMIZE_cleanup(slsiridb);
+ return;
+ }
+
+ sleep(1);
+
+ for (j = 0; j < slshards->len; j++)
+ {
+ shard = (siridb_shard_t *) slshards->data[j];
+
+ if ((shard->id - shard->id % shard->duration) + shard->duration < expi[shard->tp])
+ {
+ log_info(
+ "Shard id %" PRIu64 " (%" PRIu8 ") is expired "
+ "and will be dropped",
+ shard->id, shard->flags);
+ siridb_shard_drop(shard, siridb);
+ }
+ else if (!siri_err &&
+ optimize.status != SIRI_OPTIMIZE_CANCELLED &&
+ ((shard->flags & SIRIDB_SHARD_NEED_OPTIMIZE) ||
+ ((!(shard->flags & SIRIDB_SHARD_IS_COMPRESSED)) == c)) &&
+ (~shard->flags & SIRIDB_SHARD_IS_REMOVED))
+ {
+ log_info("Start optimizing shard id %" PRIu64 " (%" PRIu8 ")",
+ shard->id, shard->flags);
+ if (siridb_shard_optimize(shard, siridb) == 0)
+ {
+ log_info("Finished optimizing shard id %" PRIu64,
+ shard->id);
+ }
+ else
+ {
+ /* signal is raised */
+ log_critical(
+ "Optimizing shard id %" PRIu64 " has failed with a "
+ "critical error", shard->id);
+ }
+
+ if (optimize.idx_fn != NULL)
+ {
+ log_debug(
+ "Cleanup temporary index file: '%s'",
+ optimize.idx_fn);
+ if (optimize.idx_fp != NULL)
+ {
+ fclose(optimize.idx_fp);
+ optimize.idx_fp = NULL;
+ }
+ if (unlink(optimize.idx_fn))
+ {
+ log_error(
+ "Failed to remove file: '%s'",
+ optimize.idx_fn);
+ }
+ free(optimize.idx_fn);
+ optimize.idx_fn = NULL;
+ }
+ }
+
+ /* decrement ref for the shard which was incremented earlier */
+ siridb_shard_decref(shard);
+ }
+
+ vec_free(slshards);
+
+ if (siri_optimize_wait() == SIRI_OPTIMIZE_CANCELLED)
+ {
+ break;
+ }
+ log_debug("Finished optimizing database '%s'", siridb->dbname);
+ }
+ OPTIMIZE_cleanup(slsiridb);
+}
+
+static void OPTIMIZE_cleanup(vec_t * slsiridb)
+{
+ if (slsiridb != NULL)
+ {
+ siridb_t * siridb;
+ size_t i;
+
+ for (i = 0; i < slsiridb->len; i++)
+ {
+ siridb = (siridb_t *) slsiridb->data[i];
+ siridb_decref(siridb);
+ }
+
+ vec_free(slsiridb);
+ }
+}
+
+static void OPTIMIZE_work_finish(
+ uv_work_t * work __attribute__((unused)),
+ int status)
+{
+ /*
+ * Main Thread
+ */
+
+ if (Logger.level <= LOGGER_INFO)
+ {
+ log_info("Finished optimize task in %d seconds with status: %d",
+ time(NULL) - optimize.start,
+ status);
+ }
+
+ /* reset optimize status to pending if and only if the status is RUNNING */
+ if (optimize.status == SIRI_OPTIMIZE_RUNNING)
+ {
+ optimize.status = SIRI_OPTIMIZE_PENDING;
+ }
+}
+
+/*
+ * Start the optimize task. (will start a new thread performing the work)
+ */
+static void OPTIMIZE_cb(uv_timer_t * handle __attribute__((unused)))
+{
+ /*
+ * Main Thread
+ */
+ if (optimize.status != SIRI_OPTIMIZE_PENDING)
+ {
+ log_debug("Skip optimize task because of having status: %d",
+ optimize.status);
+ return;
+ }
+
+ /* set status to RUNNING */
+ optimize.status = SIRI_OPTIMIZE_RUNNING;
+
+ /* set start time */
+ optimize.start = time(NULL);
+
+ uv_queue_work(
+ siri.loop,
+ &optimize.work,
+ OPTIMIZE_work,
+ OPTIMIZE_work_finish);
+}
--- /dev/null
+/*
+ * account.c - SiriDB Service Account.
+ */
+#include <siri/service/account.h>
+#include <stddef.h>
+#include <owcrypt/owcrypt.h>
+#include <xpath/xpath.h>
+#include <siri/siri.h>
+#include <stdarg.h>
+#include <logger/logger.h>
+#include <base64/base64.h>
+
+#define SIRI_SERVICE_ACCOUNT_SCHEMA 1
+#define FILENAME ".accounts.dat"
+
+#define DEFAULT_ACCOUNT "sa"
+#define DEFAULT_PASSWORD "siri"
+
+static int ACCOUNT_free(siri_service_account_t * account, void * args);
+static int ACCOUNT_save(siri_service_account_t * account, qp_fpacker_t * fpacker);
+static void ACCOUNT_msg(char * err_msg, char * fmt, ...);
+static int ACCOUNT_cmp(
+ siri_service_account_t * account,
+ qp_obj_t * qp_account);
+
+/*
+ * Initialize siri->accounts. Returns 0 if successful or -1 in case of an error
+ * (a signal might be raised because of qpack)
+ */
+int siri_service_account_init(siri_t * siri)
+{
+ qp_unpacker_t * unpacker;
+ qp_obj_t qp_schema;
+ qp_obj_t qp_account;
+ qp_obj_t qp_password;
+ int rc = 0;
+
+ /* get service accounts file name */
+ char fn[strlen(siri->cfg->db_path) + strlen(FILENAME) + 1];
+
+ /* initialize linked list */
+ siri->accounts = llist_new();
+ if (siri->accounts == NULL)
+ {
+ return -1;
+ }
+
+ /* make filename */
+ sprintf(fn, "%s%s", siri->cfg->db_path, FILENAME);
+
+ if (!xpath_file_exist(fn))
+ {
+ /* missing file, lets create the first account */
+ qp_account.via.raw = (unsigned char *) DEFAULT_ACCOUNT;
+ qp_account.len = 4;
+ qp_password.via.raw = (unsigned char *) DEFAULT_PASSWORD;
+ qp_password.len = 4;
+
+ return (siri_service_account_new(
+ siri,
+ &qp_account,
+ &qp_password,
+ 0,
+ NULL) || siri_service_account_save(siri, NULL));
+ }
+
+ if ((unpacker = qp_unpacker_ff(fn)) == NULL)
+ {
+ return -1; /* a signal is raised is case of a memory error */
+ }
+
+ if ( !qp_is_array(qp_next(unpacker, NULL)) ||
+ qp_next(unpacker, &qp_schema) != QP_INT64 ||
+ qp_schema.via.int64 != SIRI_SERVICE_ACCOUNT_SCHEMA)
+ {
+ log_critical("Invalid schema detected in '%s'", fn);
+ qp_unpacker_ff_free(unpacker);
+ return -1;
+ }
+
+ while ( rc == 0 &&
+ qp_is_array(qp_next(unpacker, NULL)) &&
+ qp_next(unpacker, &qp_account) == QP_RAW &&
+ qp_next(unpacker, &qp_password) == QP_RAW &&
+ qp_password.len > 12) /* old and new passwords are > 12 */
+ {
+ rc = siri_service_account_new(siri, &qp_account, &qp_password, 1, NULL);
+ }
+
+ qp_unpacker_ff_free(unpacker);
+ return rc;
+}
+
+/*
+ * Creates a new service account and returns 0 if successful. In case of
+ * an error, -1 is returned, err_msg is set.
+ *
+ * When successful, the account is added to the siri->accounts linked list.
+ *
+ * is_encrypted should be zero if the password is not encrypted yet.
+ *
+ * Note: the account will not be saved to disk. Call siri_service_account_save()
+ * to save a new service account.
+ */
+int siri_service_account_new(
+ siri_t * siri,
+ qp_obj_t * qp_account,
+ qp_obj_t * qp_password,
+ int is_encrypted,
+ char * err_msg)
+{
+ siri_service_account_t * account;
+
+ account = (siri_service_account_t *) llist_get(
+ siri->accounts,
+ (llist_cb) ACCOUNT_cmp,
+ (void *) qp_account);
+
+ if (account != NULL)
+ {
+ ACCOUNT_msg(
+ err_msg,
+ "service account '%.*s' already exists",
+ (int) qp_account->len,
+ qp_account->via.raw);
+ return -1;
+ }
+
+ if (qp_account->len < 2)
+ {
+ ACCOUNT_msg(
+ err_msg,
+ "service account name should have at least 2 characters");
+ return -1;
+ }
+
+ if (qp_password->len < 2)
+ {
+ ACCOUNT_msg(
+ err_msg,
+ "service account password should have at least 2 characters");
+ return -1;
+ }
+
+ account = malloc(sizeof(siri_service_account_t));
+ if (account == NULL)
+ {
+ ACCOUNT_msg(err_msg, "memory allocation error");
+ return -1;
+ }
+
+ account->account = strndup(
+ (const char *) qp_account->via.raw, qp_account->len);
+ account->password = strndup(
+ (const char *) qp_password->via.raw, qp_password->len);
+
+ if (!is_encrypted && account->password != NULL)
+ {
+ char encrypted[OWCRYPT_SZ];
+ char salt[OWCRYPT_SALT_SZ];
+
+ /* generate a random salt */
+ owcrypt_gen_salt(salt);
+
+ /* encrypt the accounts password */
+ owcrypt(account->password, salt, encrypted);
+
+ /* replace with encrypted password */
+ free(account->password);
+ account->password = strdup(encrypted);
+ }
+
+ if ( account->account == NULL ||
+ account->password == NULL ||
+ llist_append(siri->accounts, account))
+ {
+ ACCOUNT_msg(err_msg, "memory allocation error");
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * Returns 0 if the account/password is valid or another value if not.
+ */
+int siri_service_account_check(
+ siri_t * siri,
+ qp_obj_t * qp_account,
+ qp_obj_t * qp_password,
+ char * err_msg)
+{
+ siri_service_account_t * account;
+ char pw[OWCRYPT_SZ];
+ char * password;
+
+ account = (siri_service_account_t *) llist_get(
+ siri->accounts,
+ (llist_cb) ACCOUNT_cmp,
+ (void *) qp_account);
+
+ if (account == NULL)
+ {
+ ACCOUNT_msg(
+ err_msg,
+ "cannot find service account '%.*s'",
+ (int) qp_account->len,
+ qp_account->via.raw);
+ return -1;
+ }
+
+ password = strndup(
+ (const char *) qp_password->via.raw, qp_password->len);
+
+ if (password == NULL)
+ {
+ ACCOUNT_msg(err_msg, "memory allocation error");
+ return -1;
+ }
+
+ owcrypt(password, account->password, pw);
+ free(password);
+
+ if (strcmp(pw, account->password))
+ {
+ ACCOUNT_msg(
+ err_msg,
+ "incorrect password for service account '%.*s'",
+ (int) qp_account->len,
+ qp_account->via.raw);
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * Returns 0 if the password is successful changed or -1 if not.
+ *
+ * Note: the password change is not saved, call siri_service_account_save().
+ */
+int siri_service_account_change_password(
+ siri_t * siri,
+ qp_obj_t * qp_account,
+ qp_obj_t * qp_password,
+ char * err_msg)
+{
+ siri_service_account_t * account;
+ char encrypted[OWCRYPT_SZ];
+ char salt[OWCRYPT_SALT_SZ];
+ char * password;
+
+ account = (siri_service_account_t *) llist_get(
+ siri->accounts,
+ (llist_cb) ACCOUNT_cmp,
+ (void *) qp_account);
+
+ if (account == NULL)
+ {
+ ACCOUNT_msg(
+ err_msg,
+ "cannot find service account '%.*s'",
+ (int) qp_account->len,
+ qp_account->via.raw);
+ return -1;
+ }
+
+ password = strndup(
+ (const char *) qp_password->via.raw, qp_password->len);
+
+ if (password == NULL)
+ {
+ ACCOUNT_msg(err_msg, "memory allocation error");
+ return -1;
+ }
+
+ /* generate a random salt */
+ owcrypt_gen_salt(salt);
+
+ /* encrypt the accounts password */
+ owcrypt(password, salt, encrypted);
+
+ free(password);
+
+ password = strdup(encrypted);
+ if (password == NULL)
+ {
+ ACCOUNT_msg(err_msg, "memory allocation error");
+ return -1;
+ }
+
+ /* replace account password with new encrypted password */
+ free(account->password);
+ account->password = password;
+
+ return 0;
+}
+
+static int ACCOUNT_cmp_str(siri_service_account_t * account, char * str)
+{
+ return strcmp(account->account, str) == 0;
+}
+
+
+_Bool siri_service_account_check_basic(
+ siri_t * siri,
+ const char * data,
+ size_t n)
+{
+ siri_service_account_t * account;
+ size_t size, nn, end;
+ char * b64 = base64_decode(data, n, &size);
+ _Bool is_valid = false;
+ char pw[OWCRYPT_SZ];
+
+ for (nn = 0, end = size; nn < end; ++nn)
+ {
+ if (b64[nn] == ':')
+ {
+ b64[nn] = '\0';
+
+ if (++nn > end)
+ break;
+
+ account = (siri_service_account_t *) llist_get(
+ siri->accounts,
+ (llist_cb) ACCOUNT_cmp_str,
+ b64);
+ if (account)
+ {
+ owcrypt(b64 + nn, account->password, pw);
+ is_valid = strcmp(pw, account->password) == 0;
+ }
+ break;
+ }
+ }
+
+ free(b64);
+ return is_valid;
+}
+
+/*
+ * Returns 0 if dropped or -1 in case the account was not found.
+ *
+ * Note: accounts are not saved, call siri_service_account_save().
+ */
+int siri_service_account_drop(
+ siri_t * siri,
+ qp_obj_t * qp_account,
+ char * err_msg)
+{
+ siri_service_account_t * account;
+ account = (siri_service_account_t *) llist_remove(
+ siri->accounts,
+ (llist_cb) ACCOUNT_cmp,
+ (void *) qp_account);
+ if (account == NULL)
+ {
+ ACCOUNT_msg(
+ err_msg,
+ "cannot find service account '%.*s'",
+ (int) qp_account->len,
+ qp_account->via.raw);
+ return -1;
+ }
+
+ ACCOUNT_free(account, NULL);
+ return 0;
+}
+
+/*
+ * Destroy service accounts. siri->accounts is allowed to be NULL.
+ */
+void siri_service_account_destroy(siri_t * siri)
+{
+ if (siri->accounts != NULL)
+ {
+ llist_free_cb(siri->accounts, (llist_cb) ACCOUNT_free, NULL);
+ }
+}
+
+/*
+ * Returns 0 if successful or EOF if not.
+ */
+int siri_service_account_save(siri_t * siri, char * err_msg)
+{
+ qp_fpacker_t * fpacker;
+
+ /* get service accounts file name */
+ char fn[strlen(siri->cfg->db_path) + strlen(FILENAME) + 1];
+
+ /* make filename */
+ sprintf(fn, "%s%s", siri->cfg->db_path, FILENAME);
+
+ if (
+ /* open a new account file */
+ (fpacker = qp_open(fn, "w")) == NULL ||
+
+ /* open a new array */
+ qp_fadd_type(fpacker, QP_ARRAY_OPEN) ||
+
+ /* write the current schema */
+ qp_fadd_int64(fpacker, SIRI_SERVICE_ACCOUNT_SCHEMA) ||
+
+ /* we can and should skip this if we have no accounts to save */
+ llist_walk(siri->accounts, (llist_cb) ACCOUNT_save, fpacker) ||
+
+ /* close file pointer */
+ qp_close(fpacker))
+ {
+ if (err_msg)
+ {
+ ACCOUNT_msg(err_msg, "error saving service accounts");
+ }
+ else
+ {
+ log_error("error saving service accounts: `%s`", fn);
+ }
+ return EOF;
+ }
+ return 0;
+}
+
+/*
+ * Destroy an account.
+ */
+static int ACCOUNT_free(
+ siri_service_account_t * account,
+ void * args __attribute__((unused)))
+{
+ free(account->account);
+ free(account->password);
+ free(account);
+ return 0;
+}
+
+/*
+ * Returns 0 if successful and -1 in case an error occurred.
+ */
+static int ACCOUNT_save(siri_service_account_t * account, qp_fpacker_t * fpacker)
+{
+ int rc = 0;
+
+ rc += qp_fadd_type(fpacker, QP_ARRAY2);
+ rc += qp_fadd_string(fpacker, account->account);
+ rc += qp_fadd_string(fpacker, account->password);
+
+ return rc;
+}
+
+static void ACCOUNT_msg(char * err_msg, char * fmt, ...)
+{
+ va_list args;
+ va_start(args, fmt);
+ if (err_msg != NULL)
+ {
+ vsnprintf(err_msg, SIRI_MAX_SIZE_ERR_MSG, fmt, args);
+ }
+ else
+ {
+ log_error(fmt, args);
+ }
+ va_end(args);
+}
+
+static int ACCOUNT_cmp(
+ siri_service_account_t * account,
+ qp_obj_t * qp_account)
+{
+ size_t len = strlen(account->account);
+ return (len == qp_account->len && strncmp(
+ account->account,
+ (const char *) qp_account->via.raw,
+ len) == 0);
+}
--- /dev/null
+/*
+ * client.c - Client for expanding a SiriDB database.
+ */
+#include <string.h>
+#include <stdarg.h>
+#include <lock/lock.h>
+#include <logger/logger.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <siri/service/client.h>
+#include <siri/service/request.h>
+#include <siri/net/stream.h>
+#include <siri/net/protocol.h>
+#include <siri/net/tcp.h>
+#include <siri/db/server.h>
+
+/* 15 seconds */
+#define CLIENT_REQUEST_TIMEOUT 15000
+#define CLIENT_FLAGS_TIMEOUT 1
+#define CLIENT_FLAGS_NO_ROLLBACK 2
+#define MAX_VERSION_LEN 256
+
+enum
+{
+ CLIENT_REQUEST_INIT,
+ CLIENT_REQUEST_STATUS,
+ CLIENT_REQUEST_POOLS,
+ CLIENT_REQUEST_FILE_USERS,
+ CLIENT_REQUEST_FILE_GROUPS,
+ CLIENT_REQUEST_FILE_SERVERS,
+ CLIENT_REQUEST_FILE_DATABASE,
+ CLIENT_REQUEST_REGISTER_SERVER
+};
+
+static void CLIENT_write_cb(uv_write_t * req, int status);
+static void CLIENT_on_connect(uv_connect_t * req, int status);
+static void CLIENT_on_data(sirinet_stream_t * client, sirinet_pkg_t * pkg);
+static void CLIENT_request_timeout(uv_timer_t * handle);
+static void CLIENT_on_auth_success(siri_service_client_t * adm_client);
+static int CLIENT_resolve_dns(
+ siri_service_client_t * adm_client,
+ int ai_family,
+ char * err_msg);
+static void CLIENT_on_resolved(
+ uv_getaddrinfo_t * resolver,
+ int status,
+ struct addrinfo * res);
+static void CLIENT_err(
+ siri_service_client_t * adm_client,
+ const char * fmt,
+ ...);
+static void CLIENT_send_pkg(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg);
+static void CLIENT_on_error_msg(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg);
+static void CLIENT_on_register_server(siri_service_client_t * adm_client);
+static void CLIENT_on_file_database(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg);
+static void CLIENT_on_file_servers(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg);
+static void CLIENT_on_file_groups(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg);
+static void CLIENT_on_file_users(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg);
+static void CLIENT_on_request_pools(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg);
+static void CLIENT_on_request_status(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg);
+
+/*
+ * Initializes a request for a new pool or replica and returns 0 when
+ * successful. In case of an error, err_msg will be set and a value other than
+ * 0 is returned. (a signal can be raised.) When successful and only when
+ * successful, a response will be send to the provided client using the given
+ * pid.
+ */
+int siri_service_client_request(
+ uint16_t pid,
+ uint16_t port,
+ int pool,
+ uuid_t * uuid,
+ qp_obj_t * host,
+ qp_obj_t * username,
+ qp_obj_t * password,
+ qp_obj_t * dbname,
+ const char * dbpath,
+ sirinet_stream_t * client,
+ char * err_msg)
+{
+ siri_service_client_t * adm_client;
+ struct in_addr sa;
+ struct in6_addr sa6;
+
+ if (siri.client != NULL)
+ {
+ sprintf(err_msg, "manage socket already in use");
+ return -1;
+ }
+
+ siri.client = sirinet_stream_new(STREAM_TCP_MANAGE, &CLIENT_on_data);
+ if (siri.client == NULL)
+ {
+ sprintf(err_msg, "memory allocation error");
+ return -1;
+ }
+
+ uv_tcp_init(siri.loop, (uv_tcp_t *) siri.client->stream);
+
+ adm_client = malloc(sizeof(siri_service_client_t));
+ if (adm_client == NULL)
+ {
+ sirinet_stream_decref(siri.client);
+ sprintf(err_msg, "memory allocation error");
+ return -1;
+ }
+ adm_client->pid = pid;
+ adm_client->port = port;
+ adm_client->host = strndup(
+ (const char *) host->via.raw, host->len);
+ adm_client->username = strndup(
+ (const char *) username->via.raw, username->len);
+ adm_client->password = strndup(
+ (const char *) password->via.raw, password->len);
+ adm_client->dbname = strndup(
+ (const char *) dbname->via.raw, dbname->len);
+ adm_client->dbpath = strdup(dbpath);
+ adm_client->client = client;
+ adm_client->request = CLIENT_REQUEST_INIT;
+ adm_client->flags = 0;
+ adm_client->pool = pool;
+ memcpy(&adm_client->uuid, uuid, 16);
+
+
+ siri.client->origin = (void *) adm_client;
+
+ sirinet_stream_incref(adm_client->client);
+
+ if (adm_client->host == NULL ||
+ adm_client->username == NULL ||
+ adm_client->password == NULL ||
+ adm_client->dbname == NULL ||
+ adm_client->dbpath == NULL)
+ {
+ sirinet_stream_decref(siri.client);
+ sprintf(err_msg, "memory allocation error");
+ return -1;
+ }
+
+ if (inet_pton(AF_INET, adm_client->host, &sa))
+ {
+ /* IPv4 */
+ struct sockaddr_in dest;
+
+ uv_connect_t * req = malloc(sizeof(uv_connect_t));
+ if (req == NULL)
+ {
+ sirinet_stream_decref(siri.client);
+ sprintf(err_msg, "memory allocation error");
+ return -1;
+ }
+ log_debug(
+ "Trying to connect to '%s:%u'...",
+ adm_client->host,
+ adm_client->port);
+
+ uv_ip4_addr(adm_client->host, adm_client->port, &dest);
+ uv_tcp_connect(
+ req,
+ (uv_tcp_t *) siri.client->stream,
+ (const struct sockaddr *) &dest,
+ CLIENT_on_connect);
+ }
+ else if (inet_pton(AF_INET6, adm_client->host, &sa6))
+ {
+ /* IPv6 */
+ struct sockaddr_in6 dest6;
+
+ uv_connect_t * req = malloc(sizeof(uv_connect_t));
+ if (req == NULL)
+ {
+ sirinet_stream_decref(siri.client);
+ sprintf(err_msg, "memory allocation error");
+ return -1;
+ }
+ log_debug(
+ "Trying to connect to '%s:%u'...",
+ adm_client->host,
+ adm_client->port);
+ uv_ip6_addr(adm_client->host, adm_client->port, &dest6);
+ uv_tcp_connect(
+ req,
+ (uv_tcp_t *) siri.client->stream,
+ (const struct sockaddr *) &dest6,
+ CLIENT_on_connect);
+ }
+ else
+ {
+ if (CLIENT_resolve_dns(
+ adm_client,
+ dns_req_family_map(siri.cfg->ip_support),
+ err_msg))
+ {
+ sirinet_stream_decref(siri.client);
+ return -1; /* err_msg is set */
+ }
+ }
+ uv_timer_init(siri.loop, &siri.timer);
+ return 0;
+}
+
+/*
+ * Destroy the client request. (will be called when the socket is destroyed)
+ */
+void siri_service_client_free(siri_service_client_t * adm_client)
+{
+ if (adm_client != NULL)
+ {
+ sirinet_stream_decref(adm_client->client);
+ free(adm_client->host);
+ free(adm_client->username);
+ free(adm_client->password);
+ free(adm_client->dbname);
+ free(adm_client->dbpath);
+ free(adm_client);
+ }
+}
+/*
+ * Try to get an ip address from dns.
+ *
+ * This function should return 0 when we can start an attempt for discovering
+ * dns. When the result is not zero, CLIENT_on_resolved will not be called and
+ * err_msg is set.
+ */
+static int CLIENT_resolve_dns(
+ siri_service_client_t * adm_client,
+ int ai_family,
+ char * err_msg)
+{
+ struct addrinfo hints;
+ hints.ai_family = ai_family;
+ hints.ai_socktype = SOCK_STREAM;
+ hints.ai_protocol = IPPROTO_TCP;
+ hints.ai_flags = AI_NUMERICSERV;
+
+ uv_getaddrinfo_t * resolver = malloc(sizeof(uv_getaddrinfo_t));
+
+ if (resolver == NULL)
+ {
+ sprintf(err_msg, "memory allocation error");
+ return -1;
+ }
+
+ int result;
+ resolver->data = adm_client;
+
+ char port[6]= {'\0'};
+ sprintf(port, "%u", adm_client->port);
+
+ result = uv_getaddrinfo(
+ siri.loop,
+ resolver,
+ (uv_getaddrinfo_cb) CLIENT_on_resolved,
+ adm_client->host,
+ port,
+ &hints);
+
+ if (result)
+ {
+ snprintf(
+ err_msg,
+ SIRI_MAX_SIZE_ERR_MSG,
+ "getaddrinfo call error %s",
+ uv_err_name(result));
+ free(resolver);
+ }
+
+ return result;
+}
+
+/*
+ * Callback used to check if resolving an ip address was successful.
+ */
+static void CLIENT_on_resolved(
+ uv_getaddrinfo_t * resolver,
+ int status,
+ struct addrinfo * res)
+{
+ siri_service_client_t * adm_client = (siri_service_client_t *) resolver->data;
+
+ if (status < 0)
+ {
+ CLIENT_err(
+ adm_client,
+ "cannot resolve ip address for '%s' (error: %s)",
+ adm_client->host,
+ uv_err_name(status));
+ }
+ else
+ {
+ uv_connect_t * req = malloc(sizeof(uv_connect_t));
+ if (req == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ else
+ {
+ uv_tcp_connect(
+ req,
+ (uv_tcp_t *) siri.client->stream,
+ (const struct sockaddr *) res->ai_addr,
+ CLIENT_on_connect);
+ }
+ }
+
+ uv_freeaddrinfo(res);
+ free(resolver);
+}
+
+/*
+ * In case a client request fails, this function should be called to end
+ * the request. A package with the error message will be send to the client.
+ */
+static void CLIENT_err(
+ siri_service_client_t * adm_client,
+ const char * fmt,
+ ...)
+{
+ char err_msg[SIRI_MAX_SIZE_ERR_MSG];
+
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(err_msg, SIRI_MAX_SIZE_ERR_MSG, fmt, args);
+ va_end(args);
+
+ sirinet_pkg_t * package = sirinet_pkg_err(
+ adm_client->pid,
+ strlen(err_msg),
+ CPROTO_ERR_SERVICE,
+ err_msg);
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(adm_client->client, package);
+ }
+
+ log_error(err_msg);
+
+ if (~adm_client->flags & CLIENT_FLAGS_NO_ROLLBACK)
+ {
+ siri_service_request_rollback(adm_client->dbpath);
+ }
+
+ sirinet_stream_decref(siri.client);
+
+ uv_close((uv_handle_t *) &siri.timer, NULL);
+}
+
+/*
+ * Send a package to the 'other' SiriDB server. This function can be used for
+ * sending api requests for all database information required to create the
+ * database. Note that all packages are send with the same pid so packages
+ * should be send in sequence, not parallel.
+ *
+ * Note: pkg will be freed by calling this function.
+ */
+static void CLIENT_send_pkg(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg)
+{
+ uv_write_t * req = malloc(sizeof(uv_write_t));
+ if (req == NULL)
+ {
+ free(pkg);
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ req->data = adm_client;
+
+ adm_client->pkg = pkg;
+
+ /* set the correct check bit */
+ pkg->checkbit = pkg->tp ^ 255;
+
+ uv_timer_start(
+ &siri.timer,
+ CLIENT_request_timeout,
+ CLIENT_REQUEST_TIMEOUT,
+ 0);
+
+ siri.timer.data = adm_client;
+
+ uv_buf_t wrbuf = uv_buf_init(
+ (char *) pkg,
+ sizeof(sirinet_pkg_t) + pkg->len);
+
+ uv_write(
+ req,
+ siri.client->stream,
+ &wrbuf,
+ 1,
+ CLIENT_write_cb);
+}
+
+/*
+ * Write call-back.
+ */
+static void CLIENT_write_cb(uv_write_t * req, int status)
+{
+ siri_service_client_t * adm_client = (siri_service_client_t *) req->data;
+
+ if (status)
+ {
+ uv_timer_stop(&siri.timer);
+ CLIENT_err(adm_client, "socket write error: %s", uv_strerror(status));
+ }
+
+ free(adm_client->pkg);
+ free(req);
+}
+
+/*
+ * Called when a connection to the 'other' siridb server is made.
+ * An authentication request will be send by user/password since this will be
+ * using the client connection. (a signal can be raised)
+ */
+static void CLIENT_on_connect(uv_connect_t * req, int status)
+{
+ sirinet_stream_t * client = req->handle->data;
+ siri_service_client_t * adm_client = client->origin;
+
+ if (status == 0)
+ {
+ log_debug(
+ "Connected to SiriDB server: '%s:%u', "
+ "sending authentication request",
+ adm_client->host, adm_client->port);
+
+ uv_read_start(
+ req->handle,
+ sirinet_stream_alloc_buffer,
+ sirinet_stream_on_data);
+
+ sirinet_pkg_t * pkg;
+ qp_packer_t * packer = sirinet_packer_new(512);
+
+ if (packer == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ else
+ {
+ if (qp_add_type(packer, QP_ARRAY3) ||
+ qp_add_string(packer, adm_client->username) ||
+ qp_add_string(packer, adm_client->password) ||
+ qp_add_string(packer, adm_client->dbname))
+ {
+ qp_packer_free(packer);
+ }
+ else
+ {
+ pkg = sirinet_packer2pkg(packer, 0, CPROTO_REQ_AUTH);
+ CLIENT_send_pkg(adm_client, pkg);
+ }
+
+ }
+ }
+ else
+ {
+ CLIENT_err(
+ adm_client,
+ "connecting to server '%s:%u' failed with error: %s",
+ adm_client->host,
+ adm_client->port,
+ uv_strerror(status));
+ }
+ free(req);
+}
+
+/*
+ * on-data call-back function.
+ */
+static void CLIENT_on_data(sirinet_stream_t * client, sirinet_pkg_t * pkg)
+{
+ siri_service_client_t * adm_client = client->origin;
+ log_debug(
+ "Client response received (pid: %" PRIu16
+ ", len: %" PRIu32 ", tp: %s)",
+ pkg->pid,
+ pkg->len,
+ sirinet_cproto_server_str(pkg->tp));
+
+ if (adm_client->flags & CLIENT_FLAGS_TIMEOUT)
+ {
+ log_error("Client response received which was timed-out earlier");
+ }
+ else
+ {
+ uv_timer_stop(&siri.timer);
+
+ switch ((cproto_server_t) pkg->tp)
+ {
+ case CPROTO_RES_AUTH_SUCCESS:
+ CLIENT_on_auth_success(adm_client);
+ break;
+ case CPROTO_RES_QUERY:
+ switch (adm_client->request)
+ {
+ case CLIENT_REQUEST_STATUS:
+ CLIENT_on_request_status(adm_client, pkg);
+ break;
+ case CLIENT_REQUEST_POOLS:
+ CLIENT_on_request_pools(adm_client, pkg);
+ break;
+ default:
+ CLIENT_err(adm_client, "unexpected query response");
+ }
+ break;
+ case CPROTO_RES_FILE:
+ switch (adm_client->request)
+ {
+ case CLIENT_REQUEST_FILE_USERS:
+ CLIENT_on_file_users(adm_client, pkg);
+ break;
+ case CLIENT_REQUEST_FILE_GROUPS:
+ CLIENT_on_file_groups(adm_client, pkg);
+ break;
+ case CLIENT_REQUEST_FILE_SERVERS:
+ CLIENT_on_file_servers(adm_client, pkg);
+ break;
+ case CLIENT_REQUEST_FILE_DATABASE:
+ CLIENT_on_file_database(adm_client, pkg);
+ break;
+ default:
+ CLIENT_err(adm_client, "unexpected query response");
+ }
+ break;
+ case CPROTO_RES_ACK:
+ switch (adm_client->request)
+ {
+ case CLIENT_REQUEST_REGISTER_SERVER:
+ CLIENT_on_register_server(adm_client);
+ break;
+ default:
+ CLIENT_err(adm_client, "unexpected query response");
+ }
+ break;
+ case CPROTO_ERR_AUTH_CREDENTIALS:
+ CLIENT_err(
+ adm_client,
+ "invalid credentials for database '%s' on server '%s:%u'",
+ adm_client->dbname,
+ adm_client->host,
+ adm_client->port);
+ break;
+ case CPROTO_ERR_AUTH_UNKNOWN_DB:
+ CLIENT_err(
+ adm_client,
+ "database '%s' does not exist on server '%s:%u'",
+ adm_client->dbname,
+ adm_client->host,
+ adm_client->port);
+ break;
+ case CPROTO_ERR_MSG:
+ case CPROTO_ERR_QUERY:
+ case CPROTO_ERR_INSERT:
+ case CPROTO_ERR_SERVER:
+ case CPROTO_ERR_POOL:
+ case CPROTO_ERR_USER_ACCESS:
+ CLIENT_on_error_msg(adm_client, pkg);
+ break;
+ default:
+ CLIENT_err(
+ adm_client,
+ "unexpected response (%u) received from server '%s:%u'",
+ pkg->tp,
+ adm_client->host,
+ adm_client->port);
+ }
+ }
+}
+
+/*
+ * Called when register server was successful.
+ */
+static void CLIENT_on_register_server(siri_service_client_t * adm_client)
+{
+ sirinet_pkg_t * package = sirinet_pkg_new(
+ adm_client->pid,
+ 0,
+ CPROTO_ACK_SERVICE,
+ NULL);
+
+ if (package != NULL)
+ {
+ sirinet_pkg_send(adm_client->client, package);
+ }
+
+ log_info(
+ "Finished registering server on database '%s'",
+ adm_client->dbname);
+
+ sirinet_stream_decref(siri.client);
+ uv_close((uv_handle_t *) &siri.timer, NULL);
+}
+
+/*
+ * Called when database.dat is received.
+ */
+static void CLIENT_on_file_database(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg)
+{
+ qp_fpacker_t * fpacker;
+ qp_unpacker_t unpacker;
+ qp_obj_t
+ qp_uuid,
+ qp_schema,
+ qp_dbname,
+ qp_time_precision,
+ qp_buffer_size,
+ qp_duration_num,
+ qp_duration_log,
+ qp_timezone,
+ qp_drop_threshold,
+ qp_points_limit,
+ qp_list_limit,
+ qp_exp_log,
+ qp_exp_num;
+ siridb_t * siridb;
+ int rc;
+ /* 13 = strlen("database.dat")+1 */
+ char fn[strlen(adm_client->dbpath) + 13];
+ sprintf(fn, "%sdatabase.dat", adm_client->dbpath);
+
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+
+ if (!qp_is_array(qp_next(&unpacker, NULL)) ||
+ qp_next(&unpacker, &qp_schema) != QP_INT64 ||
+ qp_next(&unpacker, &qp_uuid) != QP_RAW || qp_uuid.len != 16 ||
+ qp_next(&unpacker, &qp_dbname) != QP_RAW ||
+ qp_next(&unpacker, &qp_time_precision) != QP_INT64 ||
+ qp_next(&unpacker, &qp_buffer_size) != QP_INT64 ||
+ qp_next(&unpacker, &qp_duration_num) != QP_INT64 ||
+ qp_next(&unpacker, &qp_duration_log) != QP_INT64 ||
+ qp_next(&unpacker, &qp_timezone) != QP_RAW ||
+ qp_next(&unpacker, &qp_drop_threshold) != QP_DOUBLE)
+ {
+ CLIENT_err(adm_client, "invalid database file received");
+ return;
+ }
+
+ /* list and points limit require at least schema 1 */
+ (void) qp_next(&unpacker, &qp_points_limit);
+ (void) qp_next(&unpacker, &qp_list_limit);
+
+ /* this is the tee pipe name when schema is >= 5 */
+ (void) qp_next(&unpacker, &qp_exp_log);
+ (void) qp_next(&unpacker, &qp_exp_num);
+
+ /* these are the expiration times when schema is >= 6 */
+
+
+ if ((fpacker = qp_open(fn, "w")) == NULL)
+ {
+ CLIENT_err(adm_client, "cannot write or create file: %s", fn);
+ return;
+ }
+
+ rc = ( qp_fadd_type(fpacker, QP_ARRAY_OPEN) ||
+ qp_fadd_int64(fpacker, SIRIDB_SCHEMA) ||
+ qp_fadd_raw(fpacker, (const unsigned char *) adm_client->uuid, 16) ||
+ qp_fadd_raw(fpacker, qp_dbname.via.raw, qp_dbname.len) ||
+ qp_fadd_int64(fpacker, qp_time_precision.via.int64) ||
+ qp_fadd_int64(fpacker, qp_buffer_size.via.int64) ||
+ qp_fadd_int64(fpacker, qp_duration_num.via.int64) ||
+ qp_fadd_int64(fpacker, qp_duration_log.via.int64) ||
+ qp_fadd_raw(fpacker, qp_timezone.via.raw, qp_timezone.len) ||
+ qp_fadd_double(fpacker, qp_drop_threshold.via.real) ||
+ qp_fadd_int64(fpacker, qp_points_limit.tp == QP_INT64
+ ? qp_points_limit.via.int64
+ : DEF_SELECT_POINTS_LIMIT) ||
+ qp_fadd_int64(fpacker, qp_list_limit.tp == QP_INT64
+ ? qp_list_limit.via.int64
+ : DEF_LIST_LIMIT) ||
+ qp_fadd_type(fpacker, QP_NULL) ||
+ qp_fadd_int64(fpacker, qp_exp_log.tp == QP_INT64
+ ? qp_exp_log.via.int64
+ : 0) ||
+ qp_fadd_int64(fpacker, qp_exp_num.tp == QP_INT64
+ ? qp_exp_num.via.int64
+ : 0) ||
+ qp_fadd_type(fpacker, QP_ARRAY_CLOSE) ||
+ qp_close(fpacker));
+
+ if (rc != 0)
+ {
+ CLIENT_err(adm_client, "cannot write or create file: %s", fn);
+ return;
+ }
+
+ siridb = siridb_new(adm_client->dbpath, LOCK_QUIT_IF_EXIST);
+
+ if (siridb == NULL)
+ {
+ CLIENT_err(adm_client, "error loading database");
+ return;
+ }
+
+ /* roll-back is not possible anymore */
+ adm_client->flags |= CLIENT_FLAGS_NO_ROLLBACK;
+
+ siridb->server->flags |= SERVER_FLAG_RUNNING;
+
+ /* Force one heart-beat */
+ siri_heartbeat_force();
+
+ sirinet_pkg_t * package;
+ qp_packer_t * packer = sirinet_packer_new(512);
+
+ if (packer == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ return;
+ }
+
+ adm_client->request = CLIENT_REQUEST_REGISTER_SERVER;
+
+ if (qp_add_type(packer, QP_ARRAY4) ||
+ qp_add_raw(packer, (const unsigned char *) &adm_client->uuid, 16) ||
+ qp_add_string(packer, siri.cfg->server_address) ||
+ qp_add_int64(packer, (int64_t) siri.cfg->listen_backend_port) ||
+ qp_add_int64(packer, (int64_t) adm_client->pool))
+ {
+ qp_packer_free(packer);
+ CLIENT_err(adm_client, "memory allocation error");
+ return;
+ }
+
+ package = sirinet_packer2pkg(packer, 0, CPROTO_REQ_REGISTER_SERVER);
+ CLIENT_send_pkg(adm_client, package);
+}
+
+/*
+ * Called when servers.dat is received.
+ */
+static void CLIENT_on_file_servers(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg)
+{
+ FILE * fp;
+ qp_unpacker_t unpacker;
+ qp_types_t tp;
+ int rc, n, close_num;
+ /* 12 = strlen("servers.dat") + 1 */
+ char fn[strlen(adm_client->dbpath) + 12];
+ sprintf(fn, "%sservers.dat", adm_client->dbpath);
+
+ fp = fopen(fn, "w");
+ if (fp == NULL)
+ {
+ CLIENT_err(adm_client, "cannot write or create file: %s", fn);
+ return;
+ }
+
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ tp = qp_next(&unpacker, NULL);
+ if (tp >= QP_ARRAY0 && tp <= QP_ARRAY5)
+ {
+ pkg->data[0] = QP_ARRAY_OPEN;
+ }
+ else if (tp != QP_ARRAY_OPEN)
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+
+ /* schema checking is not required at this moment but can be done here */
+ qp_next(&unpacker, NULL);
+
+ tp = qp_next(&unpacker, NULL);
+
+ if (!qp_is_array(tp))
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+
+ close_num = (tp == QP_ARRAY_OPEN) ? 1 : 0;
+
+ /* trim closing */
+ for (n = pkg->len;
+ (unsigned char) pkg->data[n - 1] == QP_ARRAY_CLOSE;
+ n--);
+
+ rc = (fwrite(pkg->data, n, 1, fp) == 1) ? 0 : EOF;
+
+ if (close_num)
+ {
+ rc += qp_fadd_type(fp, QP_ARRAY_CLOSE);
+ }
+
+ rc += qp_fadd_type(fp, QP_ARRAY4);
+ rc += qp_fadd_raw(fp, (const unsigned char *) &adm_client->uuid, 16);
+ rc += qp_fadd_string(fp, siri.cfg->server_address);
+ rc += qp_fadd_int64(fp, (int64_t) siri.cfg->listen_backend_port);
+ rc += qp_fadd_int64(fp, (int64_t) adm_client->pool);
+ rc += fclose(fp);
+
+ if (rc)
+ {
+ CLIENT_err(adm_client, "cannot write or create file: %s", fn);
+ }
+ else
+ {
+ sirinet_pkg_t * package;
+ adm_client->request = CLIENT_REQUEST_FILE_DATABASE;
+ package = sirinet_pkg_new(0, 0, CPROTO_REQ_FILE_DATABASE, NULL);
+ if (package == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ else
+ {
+ CLIENT_send_pkg(adm_client, package);
+ }
+ }
+}
+
+/*
+ * Called when groups.dat is received.
+ */
+static void CLIENT_on_file_groups(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg)
+{
+ FILE * fp;
+ /* 11 = strlen("groups.dat") + 1 */
+ char fn[strlen(adm_client->dbpath) + 11];
+ sprintf(fn, "%sgroups.dat", adm_client->dbpath);
+
+ fp = fopen(fn, "w");
+ if (fp == NULL)
+ {
+ CLIENT_err(adm_client, "cannot write or create file: %s", fn);
+ }
+ else
+ {
+ int rc = fwrite(pkg->data, pkg->len, 1, fp);
+
+ if (fclose(fp) || rc != 1)
+ {
+ CLIENT_err(adm_client, "cannot write data to file: %s", fn);
+ }
+ else
+ {
+ adm_client->request = CLIENT_REQUEST_FILE_SERVERS;
+ sirinet_pkg_t * package =
+ sirinet_pkg_new(0, 0, CPROTO_REQ_FILE_SERVERS, NULL);
+ if (package == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ else
+ {
+ CLIENT_send_pkg(adm_client, package);
+ }
+ }
+ }
+}
+
+/*
+ * Called when users.dat is received.
+ */
+static void CLIENT_on_file_users(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg)
+{
+ FILE * fp;
+ /* 10 = strlen("users.dat") + 1 */
+ char fn[strlen(adm_client->dbpath) + 10];
+ sprintf(fn, "%susers.dat", adm_client->dbpath);
+
+ fp = fopen(fn, "w");
+ if (fp == NULL)
+ {
+ CLIENT_err(adm_client, "cannot write or create file: %s", fn);
+ }
+ else
+ {
+ int rc = fwrite(pkg->data, pkg->len, 1, fp);
+
+ if (fclose(fp) || rc != 1)
+ {
+ CLIENT_err(adm_client, "cannot write data to file: %s", fn);
+ }
+ else
+ {
+ adm_client->request = CLIENT_REQUEST_FILE_GROUPS;
+ sirinet_pkg_t * package =
+ sirinet_pkg_new(0, 0, CPROTO_REQ_FILE_GROUPS, NULL);
+ if (package == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ else
+ {
+ CLIENT_send_pkg(adm_client, package);
+ }
+ }
+ }
+}
+
+/*
+ * Called when 'list pools ...' response is received.
+ * This function will check which pool number will be assigned for a new pool
+ * or checks if a given pool for a new replica is valid.
+ */
+static void CLIENT_on_request_pools(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg)
+{
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_obj_t qp_val;
+ qp_obj_t qp_pool;
+ qp_obj_t qp_servers;
+ int columns_found = 0;
+ int validate_pool = -1;
+
+ if (!qp_is_map(qp_next(&unpacker, NULL)))
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+
+ qp_next(&unpacker, &qp_val);
+
+ while (qp_val.tp == QP_RAW)
+ {
+ if ( strncmp(
+ (const char *) qp_val.via.raw,
+ "columns",
+ qp_val.len) == 0 &&
+ qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_val) == QP_RAW &&
+ strncmp(
+ (const char *) qp_val.via.raw, "pool", qp_val.len) == 0 &&
+ qp_next(&unpacker, &qp_val) == QP_RAW &&
+ strncmp(
+ (const char *) qp_val.via.raw, "servers", qp_val.len) == 0)
+ {
+ if (qp_next(&unpacker, &qp_val) == QP_ARRAY_CLOSE)
+ {
+ qp_next(&unpacker, &qp_val);
+ }
+ columns_found = 1;
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_val.via.raw, "pools", qp_val.len) == 0 &&
+ qp_is_array(qp_next(&unpacker, NULL)))
+ {
+ qp_next(&unpacker, &qp_val);
+
+ while (qp_is_array(qp_val.tp))
+ {
+ if (qp_next(&unpacker, &qp_pool) == QP_INT64 &&
+ qp_next(&unpacker, &qp_servers) == QP_INT64)
+ {
+ if (adm_client->pool < 0)
+ {
+ /* looking for a new pool */
+ if (qp_pool.via.int64 > validate_pool)
+ {
+ validate_pool = qp_pool.via.int64;
+ }
+ }
+ else
+ {
+ if (qp_pool.via.int64 == adm_client->pool)
+ {
+ if (qp_servers.via.int64 > 1)
+ {
+ CLIENT_err(
+ adm_client,
+ "pool %d has already %" PRId64
+ " servers",
+ adm_client->pool,
+ qp_servers.via.int64);
+ return;
+ }
+ else
+ {
+ validate_pool = 0;
+ }
+ }
+ }
+ }
+ else
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+ if (qp_next(&unpacker, &qp_val) == QP_ARRAY_CLOSE)
+ {
+ qp_next(&unpacker, &qp_val);
+ }
+ }
+ if (qp_val.tp == QP_ARRAY_CLOSE)
+ {
+ qp_next(&unpacker, &qp_val);
+ }
+ continue;
+ }
+
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+
+ if (!columns_found)
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ }
+ else
+ {
+ sirinet_pkg_t * package;
+
+ if (adm_client->pool < 0)
+ {
+ if (validate_pool == -1)
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+ /* set new correct pool in case we request a new pool */
+ adm_client->pool = validate_pool + 1;
+ }
+ else if (validate_pool == -1)
+ {
+ CLIENT_err(
+ adm_client,
+ "pool %d does not exist",
+ adm_client->pool);
+ return;
+ }
+
+ adm_client->request = CLIENT_REQUEST_FILE_USERS;
+ package = sirinet_pkg_new(0, 0, CPROTO_REQ_FILE_USERS, NULL);
+ if (package == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ else
+ {
+ CLIENT_send_pkg(adm_client, package);
+ }
+ }
+}
+
+/*
+ * Called when 'list servers ...' response is received.
+ * This function will check if each current server has status running.
+ * (this is a pre-check, the final register call does check for all servers
+ * to have the running status once more)
+ */
+static void CLIENT_on_request_status(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg)
+{
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_obj_t qp_val;
+ qp_obj_t qp_name;
+ qp_obj_t qp_status;
+ qp_obj_t qp_version;
+ int columns_found = 0;
+ int servers_found = 0;
+ char version[MAX_VERSION_LEN];
+
+ if (!qp_is_map(qp_next(&unpacker, NULL)))
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+
+ qp_next(&unpacker, &qp_val);
+
+ while (qp_val.tp == QP_RAW)
+ {
+ if ( strncmp(
+ (const char *) qp_val.via.raw,
+ "columns",
+ qp_val.len) == 0 &&
+ qp_is_array(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, &qp_val) == QP_RAW &&
+ strncmp(
+ (const char *) qp_val.via.raw, "name", qp_val.len) == 0 &&
+ qp_next(&unpacker, &qp_val) == QP_RAW &&
+ strncmp(
+ (const char *) qp_val.via.raw,
+ "status",
+ qp_val.len) == 0 &&
+ qp_next(&unpacker, &qp_val) == QP_RAW &&
+ strncmp(
+ (const char *) qp_val.via.raw, "version", qp_val.len) == 0)
+ {
+ if (qp_next(&unpacker, &qp_val) == QP_ARRAY_CLOSE)
+ {
+ qp_next(&unpacker, &qp_val);
+ }
+ columns_found = 1;
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_val.via.raw,
+ "servers",
+ qp_val.len) == 0 &&
+ qp_is_array(qp_next(&unpacker, NULL)))
+ {
+ qp_next(&unpacker, &qp_val);
+
+ while (qp_is_array(qp_val.tp))
+ {
+ if (qp_next(&unpacker, &qp_name) == QP_RAW &&
+ qp_next(&unpacker, &qp_status) == QP_RAW &&
+ qp_next(&unpacker, &qp_version) == QP_RAW &&
+ qp_version.len < MAX_VERSION_LEN)
+ {
+ /* copy and null terminate version */
+ memcpy(version, qp_version.via.raw, qp_version.len);
+ version[qp_version.len] = '\0';
+
+ if (siri_version_cmp(version, SIRIDB_VERSION))
+ {
+ CLIENT_err(
+ adm_client,
+ "server '%.*s' is running on version %s "
+ "while version %s is expected. (all SiriBD "
+ "servers should run the same version)",
+ (int) qp_name.len,
+ qp_name.via.raw,
+ version,
+ SIRIDB_VERSION);
+ return;
+ }
+
+ if (strncmp(
+ (const char *) qp_status.via.raw,
+ "running",
+ qp_status.len) != 0)
+ {
+ CLIENT_err(
+ adm_client,
+ "server '%.*s' has status: '%.*s'",
+ (int) qp_name.len,
+ qp_name.via.raw,
+ (int) qp_status.len,
+ qp_status.via.raw);
+ return;
+ }
+ servers_found++;
+ }
+ else
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+ if (qp_next(&unpacker, &qp_val) == QP_ARRAY_CLOSE)
+ {
+ qp_next(&unpacker, &qp_val);
+ }
+ }
+ if (qp_val.tp == QP_ARRAY_CLOSE)
+ {
+ qp_next(&unpacker, &qp_val);
+ }
+ continue;
+ }
+
+ CLIENT_err(adm_client, "invalid server status response");
+ return;
+ }
+
+ if (!servers_found || !columns_found)
+ {
+ CLIENT_err(adm_client, "invalid server status response");
+ }
+ else
+ {
+ sirinet_pkg_t * package;
+ qp_packer_t * packer = sirinet_packer_new(512);
+ if (packer == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ else
+ {
+ adm_client->request = CLIENT_REQUEST_POOLS;
+
+ /* no need to check since this will always fit */
+ qp_add_type(packer, QP_ARRAY1);
+ qp_add_string(packer, "list pools pool, servers");
+ package = sirinet_packer2pkg(packer, 0, CPROTO_REQ_QUERY);
+ CLIENT_send_pkg(adm_client, package);
+ }
+ }
+}
+
+/*
+ * Called when an error message is received.
+ */
+static void CLIENT_on_error_msg(
+ siri_service_client_t * adm_client,
+ sirinet_pkg_t * pkg)
+{
+ qp_unpacker_t unpacker;
+ qp_unpacker_init(&unpacker, pkg->data, pkg->len);
+ qp_obj_t qp_err;
+
+ if (qp_is_map(qp_next(&unpacker, NULL)) &&
+ qp_next(&unpacker, NULL) == QP_RAW &&
+ qp_next(&unpacker, &qp_err) == QP_RAW)
+ {
+ CLIENT_err(
+ adm_client,
+ "error on server '%s:%u': %.*s",
+ adm_client->host,
+ adm_client->port,
+ (int) qp_err.len,
+ qp_err.via.raw);
+ }
+ else
+ {
+ CLIENT_err(
+ adm_client,
+ "unexpected error on server '%s:%u'",
+ adm_client->host,
+ adm_client->port);
+ }
+}
+
+/*
+ * Called when authentication is successful.
+ */
+static void CLIENT_on_auth_success(siri_service_client_t * adm_client)
+{
+ sirinet_pkg_t * pkg;
+ qp_packer_t * packer = sirinet_packer_new(512);
+ if (packer == NULL)
+ {
+ CLIENT_err(adm_client, "memory allocation error");
+ }
+ else
+ {
+ adm_client->request = CLIENT_REQUEST_STATUS;
+
+ /* no need to check since this will always fit */
+ qp_add_type(packer, QP_ARRAY1);
+ qp_add_string(packer, "list servers name, status, version");
+ pkg = sirinet_packer2pkg(packer, 0, CPROTO_REQ_QUERY);
+ CLIENT_send_pkg(adm_client, pkg);
+ }
+}
+
+/*
+ * Timeout on a client request.
+ */
+static void CLIENT_request_timeout(uv_timer_t * handle)
+{
+ siri_service_client_t * adm_client = (siri_service_client_t *) handle->data;
+
+ adm_client->flags |= CLIENT_FLAGS_TIMEOUT;
+
+ CLIENT_err(adm_client, "request timeout");
+}
--- /dev/null
+/*
+ * request.c - SiriDB Service Request.
+ */
+#define PCRE2_CODE_UNIT_WIDTH 8
+
+#include <lock/lock.h>
+#include <logger/logger.h>
+#include <pcre2.h>
+#include <siri/db/buffer.h>
+#include <siri/db/reindex.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/service/account.h>
+#include <siri/service/client.h>
+#include <siri/service/request.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <stddef.h>
+#include <unistd.h>
+#include <uuid/uuid.h>
+#include <xmath/xmath.h>
+
+#define DEFAULT_TIME_PRECISION 1
+#define DEFAULT_BUFFER_SIZE 1024
+#define DEFAULT_DURATION_NUM 604800
+#define DEFAULT_DURATION_LOG 86400
+#define DB_CONF_FN "database.conf"
+#define DB_DAT_FN "database.dat"
+#define DEFAULT_CONF \
+"#\n" \
+"# Welcome to the SiriDB configuration file\n" \
+"#\n" \
+"\n" \
+"[buffer]\n" \
+"# Alternative path to save the buffer file.\n" \
+"# In case you later plan to change this location you manually need to move\n" \
+"# the buffer file to the new location.\n" \
+"# path = <buffer_path>\n" \
+"\n" \
+"# Buffer size in bytes. This size must be a multiple of 512 with a maximum\n" \
+"# of 1048576 bytes. Be careful using large values since SiriDB will require\n" \
+"# memory based on this value. A value between 1024 and 32768 is recommended.\n" \
+"# size = 1024\n"
+
+#define CHECK_DBNAME_AND_CREATE_PATH \
+ pcre_exec_ret = pcre2_match( \
+ siri.dbname_regex, \
+ (PCRE2_SPTR8) qp_dbname.via.raw, \
+ qp_dbname.len, \
+ 0, \
+ 0, \
+ siri.dbname_match_data, \
+ NULL); \
+ \
+ if (pcre_exec_ret < 0) \
+ { \
+ snprintf( \
+ err_msg, \
+ SIRI_MAX_SIZE_ERR_MSG, \
+ "invalid database name: '%.*s'", \
+ (int) qp_dbname.len, \
+ qp_dbname.via.raw); \
+ return CPROTO_ERR_SERVICE; \
+ } \
+ \
+ if (llist_get( \
+ siri.siridb_list, \
+ (llist_cb) SERVICE_find_database, \
+ &qp_dbname) != NULL) \
+ { \
+ snprintf( \
+ err_msg, \
+ SIRI_MAX_SIZE_ERR_MSG, \
+ "database name already exists: '%.*s'", \
+ (int) qp_dbname.len, \
+ qp_dbname.via.raw); \
+ return CPROTO_ERR_SERVICE; \
+ } \
+ \
+ dbpath_len = strlen(siri.cfg->db_path) + qp_dbname.len + 2; \
+ char dbpath[dbpath_len]; \
+ sprintf(dbpath, \
+ "%s%.*s/", \
+ siri.cfg->db_path, \
+ (int) qp_dbname.len, \
+ qp_dbname.via.raw); \
+ \
+ if (stat(dbpath, &st) != -1) \
+ { \
+ snprintf( \
+ err_msg, \
+ SIRI_MAX_SIZE_ERR_MSG, \
+ "database directory already exists: %s", \
+ dbpath); \
+ return CPROTO_ERR_SERVICE; \
+ } \
+ \
+ if (mkdir(dbpath, 0700) == -1) \
+ { \
+ snprintf( \
+ err_msg, \
+ SIRI_MAX_SIZE_ERR_MSG, \
+ "cannot create directory: %s", \
+ dbpath); \
+ return CPROTO_ERR_SERVICE; \
+ } \
+ \
+ char dbfn[dbpath_len + max_filename_sz]; \
+ sprintf(dbfn, "%s%s", dbpath, DB_CONF_FN); \
+ \
+ fp = fopen(dbfn, "w"); \
+ if (fp == NULL) \
+ { \
+ siri_service_request_rollback(dbpath); \
+ snprintf( \
+ err_msg, \
+ SIRI_MAX_SIZE_ERR_MSG, \
+ "cannot open file for writing: %s", \
+ dbfn); \
+ return CPROTO_ERR_SERVICE; \
+ } \
+ \
+ rc = fputs(DEFAULT_CONF, fp); \
+ \
+ if (fclose(fp) || rc < 0) \
+ { \
+ siri_service_request_rollback(dbpath); \
+ snprintf( \
+ err_msg, \
+ SIRI_MAX_SIZE_ERR_MSG, \
+ "cannot write file: %s", \
+ dbfn); \
+ return CPROTO_ERR_SERVICE; \
+ }
+
+static cproto_server_t SERVICE_on_new_account(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg);
+static cproto_server_t SERVICE_on_change_password(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg);
+static cproto_server_t SERVICE_on_drop_account(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg);
+static cproto_server_t SERVICE_on_drop_database(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg);
+static cproto_server_t SERVICE_on_new_database(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg);
+static cproto_server_t SERVICE_on_new_replica_or_pool(
+ qp_unpacker_t * qp_unpacker,
+ uint16_t pid,
+ sirinet_stream_t * client,
+ int req,
+ char * err_msg);
+static cproto_server_t SERVICE_on_get_version(
+ qp_unpacker_t * qp_unpacker,
+ qp_packer_t ** packaddr,
+ char * err_msg);
+static cproto_server_t SERVICE_on_get_accounts(
+ qp_unpacker_t * qp_unpacker,
+ qp_packer_t ** packaddr,
+ char * err_msg);
+static cproto_server_t SERVICE_on_get_databases(
+ qp_unpacker_t * qp_unpacker,
+ qp_packer_t ** packaddr,
+ char * err_msg);
+static int8_t SERVICE_time_precision(qp_obj_t * qp_time_precision);
+static int64_t SERVICE_duration(qp_obj_t * qp_duration, uint8_t time_precision);
+static int SERVICE_list_databases(siridb_t * siridb, qp_packer_t * packer);
+static int SERVICE_find_database(siridb_t * siridb, qp_obj_t * dbname);
+static int SERVICE_list_accounts(
+ siri_service_account_t * account,
+ qp_packer_t * packer);
+static void SERVICE_on_drop_database_cb(vec_t *, void *);
+
+static size_t max_filename_sz;
+
+/*
+ * Initialize service requests. (called once when initializing SiriDB)
+ */
+int siri_service_request_init(void)
+{
+ max_filename_sz = xmath_max_size(
+ 3,
+ strlen(DB_CONF_FN),
+ strlen(DB_DAT_FN),
+ strlen(REINDEX_FN));
+
+ int pcre_error_num;
+ PCRE2_SIZE pcre_error_offset;
+
+ pcre2_code * regex;
+ pcre2_match_data * match_data;
+
+ regex = pcre2_compile(
+ (PCRE2_SPTR8) "^[a-zA-Z][a-zA-Z0-9-_]{0,18}[a-zA-Z0-9]$",
+ PCRE2_ZERO_TERMINATED,
+ 0,
+ &pcre_error_num,
+ &pcre_error_offset,
+ NULL);
+ if (regex == NULL)
+ {
+ return -1;
+ }
+ match_data = pcre2_match_data_create_from_pattern(regex, NULL);
+
+ if(match_data == NULL)
+ {
+ pcre2_match_data_free(match_data);
+ pcre2_code_free(regex);
+ return -1;
+ }
+
+ siri.dbname_regex = regex;
+ siri.dbname_match_data = match_data;
+
+ return 0;
+}
+
+/*
+ * Destroy service requests. (only called when exiting SiriDB)
+ */
+void siri_service_request_destroy(void)
+{
+ pcre2_match_data_free(siri.dbname_match_data);
+ pcre2_code_free(siri.dbname_regex);
+}
+
+/*
+ * Returns CPROTO_ACK_SERVICE or CPROTO_DEFERRED when successful.
+ * In case of an error CPROTO_ERR_SERVICE can be returned in which case err_msg
+ * is set, or CPROTO_ERR_SERVICE_INVALID_REQUEST is returned.
+ */
+cproto_server_t siri_service_request(
+ int tp,
+ qp_unpacker_t * qp_unpacker,
+ qp_packer_t ** packaddr,
+ uint16_t pid,
+ sirinet_stream_t * client,
+ char * err_msg)
+{
+ switch ((service_request_t) tp)
+ {
+ case SERVICE_NEW_ACCOUNT:
+ return SERVICE_on_new_account(qp_unpacker, err_msg);
+ case SERVICE_CHANGE_PASSWORD:
+ return SERVICE_on_change_password(qp_unpacker, err_msg);
+ case SERVICE_DROP_ACCOUNT:
+ return SERVICE_on_drop_account(qp_unpacker, err_msg);
+ case SERVICE_NEW_DATABASE:
+ return SERVICE_on_new_database(qp_unpacker, err_msg);
+ case SERVICE_NEW_POOL:
+ return SERVICE_on_new_replica_or_pool(
+ qp_unpacker,
+ pid,
+ client,
+ SERVICE_NEW_POOL,
+ err_msg);
+ case SERVICE_NEW_REPLICA:
+ return SERVICE_on_new_replica_or_pool(
+ qp_unpacker,
+ pid,
+ client,
+ SERVICE_NEW_REPLICA,
+ err_msg);
+ case SERVICE_DROP_DATABASE:
+ return SERVICE_on_drop_database(qp_unpacker, err_msg);
+ case SERVICE_GET_VERSION:
+ return SERVICE_on_get_version(qp_unpacker, packaddr, err_msg);
+ case SERVICE_GET_ACCOUNTS:
+ return SERVICE_on_get_accounts(qp_unpacker, packaddr, err_msg);
+ case SERVICE_GET_DATABASES:
+ return SERVICE_on_get_databases(qp_unpacker, packaddr, err_msg);
+ default:
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+}
+
+/*
+ * Returns CPROTO_ACK_SERVICE when successful.
+ * In case of an error CPROTO_ERR_SERVICE can be returned in which case err_msg
+ * is set, or CPROTO_ERR_SERVICE_INVALID_REQUEST is returned.
+ */
+static cproto_server_t SERVICE_on_new_account(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg)
+{
+ qp_obj_t qp_key, qp_account, qp_password;
+
+ qp_account.tp = QP_HOOK;
+ qp_password.tp = QP_HOOK;
+
+ if (!qp_is_map(qp_next(qp_unpacker, NULL)))
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ while (qp_next(qp_unpacker, &qp_key) == QP_RAW)
+ {
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "account",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_account) == QP_RAW)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "password",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_password) == QP_RAW)
+ {
+ continue;
+ }
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ if (qp_account.tp == QP_HOOK || qp_password.tp == QP_HOOK)
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ return (siri_service_account_new(
+ &siri,
+ &qp_account,
+ &qp_password,
+ 0,
+ err_msg) ||
+ siri_service_account_save(&siri, err_msg)) ?
+ CPROTO_ERR_SERVICE : CPROTO_ACK_SERVICE;
+}
+
+/*
+ * Returns CPROTO_ACK_SERVICE when successful.
+ * In case of an error CPROTO_ERR_SERVICE can be returned in which case err_msg
+ * is set, or CPROTO_ERR_SERVICE_INVALID_REQUEST is returned.
+ */
+static cproto_server_t SERVICE_on_change_password(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg)
+{
+ qp_obj_t qp_key, qp_account, qp_password;
+
+ qp_account.tp = QP_HOOK;
+ qp_password.tp = QP_HOOK;
+
+ if (!qp_is_map(qp_next(qp_unpacker, NULL)))
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ while (qp_next(qp_unpacker, &qp_key) == QP_RAW)
+ {
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "account",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_account) == QP_RAW)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "password",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_password) == QP_RAW)
+ {
+ continue;
+ }
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ if (qp_account.tp == QP_HOOK || qp_password.tp == QP_HOOK)
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ return (siri_service_account_change_password(
+ &siri,
+ &qp_account,
+ &qp_password,
+ err_msg) ||
+ siri_service_account_save(&siri, err_msg)) ?
+ CPROTO_ERR_SERVICE : CPROTO_ACK_SERVICE;
+}
+
+/*
+ * Returns CPROTO_ACK_SERVICE when successful.
+ * In case of an error CPROTO_ERR_SERVICE can be returned in which case err_msg
+ * is set, or CPROTO_ERR_SERVICE_INVALID_REQUEST is returned.
+ */
+static cproto_server_t SERVICE_on_drop_account(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg)
+{
+ qp_obj_t qp_key, qp_target;
+
+ qp_target.tp = QP_HOOK;
+
+ if (!qp_is_map(qp_next(qp_unpacker, NULL)))
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ while (qp_next(qp_unpacker, &qp_key) == QP_RAW)
+ {
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "account",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_target) == QP_RAW)
+ {
+ continue;
+ }
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ if (qp_target.tp == QP_HOOK)
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ if (siri.accounts->len == 1)
+ {
+ sprintf(err_msg,
+ "at least one service account is required, "
+ "cannot drop the last service account");
+ return CPROTO_ERR_SERVICE;
+ }
+
+ return (siri_service_account_drop(
+ &siri,
+ &qp_target,
+ err_msg) ||
+ siri_service_account_save(&siri, err_msg)) ?
+ CPROTO_ERR_SERVICE : CPROTO_ACK_SERVICE;
+}
+
+/*
+ * Returns CPROTO_ACK_SERVICE when successful.
+ * In case of an error CPROTO_ERR_SERVICE can be returned in which case err_msg
+ * is set, or CPROTO_ERR_SERVICE_INVALID_REQUEST is returned.
+ */
+static cproto_server_t SERVICE_on_drop_database(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg)
+{
+ _Bool ignore_offline;
+ siridb_t * siridb;
+ qp_obj_t qp_key, qp_target, qp_ignore_offline;
+ sirinet_pkg_t * pkg;
+ vec_t * servers;
+
+ qp_target.tp = QP_HOOK;
+ qp_ignore_offline.tp = QP_HOOK;
+
+ if (!qp_is_map(qp_next(qp_unpacker, NULL)))
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ while (qp_next(qp_unpacker, &qp_key) == QP_RAW)
+ {
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "database",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_target) == QP_RAW)
+ {
+ continue;
+ }
+
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "ignore_offline",
+ qp_key.len) == 0 &&
+ qp_is_bool(qp_next(qp_unpacker, &qp_ignore_offline)))
+ {
+ continue;
+ }
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ if (qp_target.tp == QP_HOOK)
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ ignore_offline = (
+ qp_ignore_offline.tp != QP_HOOK &&
+ qp_ignore_offline.tp == QP_TRUE
+ );
+
+ siridb = siridb_get_by_qp(siri.siridb_list, &qp_target);
+ if (siridb == NULL)
+ {
+ sprintf(err_msg, "cannot find database '%.*s'",
+ (int) qp_target.len, (char *) qp_target.via.raw);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ if (!ignore_offline && !siridb_servers_online(siridb))
+ {
+ sprintf(err_msg,
+ "at least one server is off-line, "
+ "set `ignore_offline` to true if you want to "
+ "ignore off-line servers");
+ return CPROTO_ERR_SERVICE;
+ }
+
+ pkg = sirinet_pkg_new(0, 0, BPROTO_DROP_DATABASE, NULL);
+ servers = siridb_servers_other2vec(siridb);
+ if (pkg == NULL || servers == NULL)
+ {
+ free(pkg);
+ vec_free(servers);
+ sprintf(err_msg, "memory allocation error");
+ return CPROTO_ERR_SERVICE;
+ }
+
+ siridb_servers_send_pkg(
+ servers,
+ pkg,
+ 0,
+ SERVICE_on_drop_database_cb,
+ NULL);
+
+ siridb_drop(siridb);
+ vec_free(servers);
+
+ return CPROTO_ACK_SERVICE;
+}
+
+/*
+ * Returns CPROTO_ACK_SERVICE when successful.
+ * In case of an error CPROTO_ERR_SERVICE can be returned in which case err_msg
+ * is set, or CPROTO_ERR_SERVICE_INVALID_REQUEST is returned.
+ */
+static cproto_server_t SERVICE_on_new_database(
+ qp_unpacker_t * qp_unpacker,
+ char * err_msg)
+{
+ FILE * fp;
+ qp_obj_t
+ qp_key,
+ qp_dbname,
+ qp_time_precision,
+ qp_buffer_size,
+ qp_duration_num,
+ qp_duration_log;
+ size_t dbpath_len;
+ int pcre_exec_ret;
+ int rc;
+ struct stat st;
+ int8_t time_precision;
+ int64_t buffer_size, duration_num, duration_log;
+ siridb_t * siridb;
+ uuid_t uuid;
+
+ memset(&st, 0, sizeof(struct stat));
+
+ if (siri.siridb_list->len == MAX_NUMBER_DB)
+ {
+ sprintf(err_msg,
+ "maximum number of databases is reached (%zd)",
+ siri.siridb_list->len);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ qp_dbname.tp = QP_HOOK;
+ qp_time_precision.tp = QP_HOOK;
+ qp_buffer_size.tp = QP_HOOK;
+ qp_duration_num.tp = QP_HOOK;
+ qp_duration_log.tp = QP_HOOK;
+
+ if (!qp_is_map(qp_next(qp_unpacker, NULL)))
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ while (qp_next(qp_unpacker, &qp_key) == QP_RAW)
+ {
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "dbname",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_dbname) == QP_RAW)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "time_precision",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_time_precision) == QP_RAW)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "buffer_size",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_buffer_size) == QP_INT64)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "duration_num",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_duration_num) == QP_RAW)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "duration_log",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_duration_log) == QP_RAW)
+ {
+ continue;
+ }
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ if (qp_dbname.tp == QP_HOOK)
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ time_precision = (qp_time_precision.tp == QP_HOOK) ?
+ DEFAULT_TIME_PRECISION : SERVICE_time_precision(&qp_time_precision);
+ if (time_precision == -1)
+ {
+ snprintf(
+ err_msg,
+ SIRI_MAX_SIZE_ERR_MSG,
+ "invalid time precision: '%.*s' (expecting s, ms, us or ns)",
+ (int) qp_time_precision.len,
+ qp_time_precision.via.raw);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ duration_num = (qp_duration_num.tp == QP_HOOK) ?
+ DEFAULT_DURATION_NUM * xmath_ipow(1000, time_precision):
+ SERVICE_duration(&qp_duration_num, time_precision);
+
+ if (duration_num == -1)
+ {
+ snprintf(
+ err_msg,
+ SIRI_MAX_SIZE_ERR_MSG,
+ "invalid number duration: '%.*s' "
+ "(valid examples: 6h, 2d or 1w)",
+ (int) qp_duration_num.len,
+ qp_duration_num.via.raw);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ duration_log = (qp_duration_log.tp == QP_HOOK) ?
+ DEFAULT_DURATION_LOG * xmath_ipow(1000, time_precision):
+ SERVICE_duration(&qp_duration_log, time_precision);
+
+ if (duration_log == -1)
+ {
+ snprintf(
+ err_msg,
+ SIRI_MAX_SIZE_ERR_MSG,
+ "invalid log duration: '%.*s' "
+ "(valid examples: 6h, 2d or 1w)",
+ (int) qp_duration_log.len,
+ qp_duration_log.via.raw);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ buffer_size = (qp_buffer_size.tp == QP_HOOK) ?
+ DEFAULT_BUFFER_SIZE : qp_buffer_size.via.int64;
+
+ if (!siridb_buffer_is_valid_size(buffer_size))
+ {
+ sprintf(err_msg,
+ "invalid buffer size: %" PRId64
+ " (expecting a multiple of 512 with a maximum of %" PRId64 ")",
+ buffer_size,
+ (int64_t) MAX_BUFFER_SZ);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ CHECK_DBNAME_AND_CREATE_PATH
+
+ sprintf(dbfn, "%s%s", dbpath, DB_DAT_FN);
+ fp = qp_open(dbfn, "w");
+ if (fp == NULL)
+ {
+ siri_service_request_rollback(dbpath);
+ snprintf(
+ err_msg,
+ SIRI_MAX_SIZE_ERR_MSG,
+ "cannot open file for writing: %s",
+ dbfn);
+ return CPROTO_ERR_SERVICE;
+ }
+ rc = 0;
+ uuid_generate(uuid);
+
+ if (qp_fadd_type(fp, QP_ARRAY_OPEN) ||
+ qp_fadd_int64(fp, SIRIDB_SCHEMA) ||
+ qp_fadd_raw(fp, (const unsigned char *) uuid, 16) ||
+ qp_fadd_raw(fp, qp_dbname.via.raw, qp_dbname.len) ||
+ qp_fadd_int64(fp, time_precision) ||
+ qp_fadd_int64(fp, buffer_size) ||
+ qp_fadd_int64(fp, duration_num) ||
+ qp_fadd_int64(fp, duration_log) ||
+ qp_fadd_string(fp, "NAIVE") ||
+ qp_fadd_double(fp, DEF_DROP_THRESHOLD) ||
+ qp_fadd_int64(fp, DEF_SELECT_POINTS_LIMIT) ||
+ qp_fadd_int64(fp, DEF_LIST_LIMIT) ||
+ qp_fadd_type(fp, QP_NULL) ||
+ qp_fadd_int64(fp, 0) ||
+ qp_fadd_int64(fp, 0) ||
+ qp_fadd_type(fp, QP_ARRAY_CLOSE))
+ {
+ rc = -1;
+ }
+
+ if (qp_close(fp) || rc == -1)
+ {
+ siri_service_request_rollback(dbpath);
+ snprintf(
+ err_msg,
+ SIRI_MAX_SIZE_ERR_MSG,
+ "cannot write file: %s",
+ dbfn);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ siridb = siridb_new(dbpath, LOCK_QUIT_IF_EXIST);
+ if (siridb == NULL)
+ {
+ siri_service_request_rollback(dbpath);
+ sprintf(err_msg, "error loading database");
+ return CPROTO_ERR_SERVICE;
+ }
+
+ siridb->server->flags |= SERVER_FLAG_RUNNING;
+
+ /* Force one heart-beat */
+ siri_heartbeat_force();
+
+ return CPROTO_ACK_SERVICE;
+}
+
+/*
+ * Returns CPROTO_DEFERRED when successful.
+ * In case of an error CPROTO_ERR_SERVICE can be returned in which case err_msg
+ * is set, or CPROTO_ERR_SERVICE_INVALID_REQUEST is returned.
+ */
+static cproto_server_t SERVICE_on_new_replica_or_pool(
+ qp_unpacker_t * qp_unpacker,
+ uint16_t pid,
+ sirinet_stream_t * client,
+ int req,
+ char * err_msg)
+{
+ FILE * fp;
+ qp_obj_t
+ qp_key,
+ qp_dbname,
+ qp_pool,
+ qp_host,
+ qp_port,
+ qp_username,
+ qp_password;
+ size_t dbpath_len;
+ int pcre_exec_ret;
+ int rc;
+ struct stat st;
+ uint16_t port;
+ uuid_t uuid;
+
+ memset(&st, 0, sizeof(struct stat));
+
+ if (siri.siridb_list->len == MAX_NUMBER_DB)
+ {
+ sprintf(err_msg,
+ "maximum number of databases is reached (%zd)",
+ siri.siridb_list->len);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ qp_dbname.tp = QP_HOOK;
+ qp_pool.tp = QP_HOOK;
+ qp_host.tp = QP_HOOK;
+ qp_port.tp = QP_HOOK;
+ qp_username.tp = QP_HOOK;
+ qp_password.tp = QP_HOOK;
+
+ if (!qp_is_map(qp_next(qp_unpacker, &qp_key)))
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ while (qp_next(qp_unpacker, &qp_key) == QP_RAW)
+ {
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "dbname",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_dbname) == QP_RAW)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw, "pool", qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_pool) == QP_INT64)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw, "host", qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_host) == QP_RAW)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw, "port", qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_port) == QP_INT64)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "username",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_username) == QP_RAW)
+ {
+ continue;
+ }
+ if ( strncmp(
+ (const char *) qp_key.via.raw,
+ "password",
+ qp_key.len) == 0 &&
+ qp_next(qp_unpacker, &qp_password) == QP_RAW)
+ {
+ continue;
+ }
+
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ if (qp_dbname.tp == QP_HOOK ||
+ (
+ (req == SERVICE_NEW_POOL && qp_pool.tp != QP_HOOK) ||
+ (req == SERVICE_NEW_REPLICA && qp_pool.tp == QP_HOOK)
+ ) ||
+ qp_host.tp == QP_HOOK ||
+ qp_port.tp == QP_HOOK ||
+ qp_username.tp == QP_HOOK ||
+ qp_password.tp == QP_HOOK)
+ {
+ return CPROTO_ERR_SERVICE_INVALID_REQUEST;
+ }
+
+ if (qp_port.via.int64 < 1 || qp_port.via.int64 > 65535)
+ {
+ sprintf(err_msg,
+ "invalid port number: %" PRId64
+ " (expecting a value between 0 and 65536)",
+ qp_port.via.int64);
+ return CPROTO_ERR_SERVICE;
+ }
+
+ port = (uint16_t) qp_port.via.int64;
+ uuid_generate(uuid);
+
+ CHECK_DBNAME_AND_CREATE_PATH
+
+ if (req == SERVICE_NEW_POOL)
+ {
+ sprintf(dbfn, "%s%s", dbpath, REINDEX_FN);
+ fp = fopen(dbfn, "w");
+ if (fp == NULL || fclose(fp))
+ {
+ siri_service_request_rollback(dbpath);
+ snprintf(
+ err_msg,
+ SIRI_MAX_SIZE_ERR_MSG,
+ "cannot open file for writing: %s",
+ dbfn);
+ return CPROTO_ERR_SERVICE;
+ }
+ }
+
+ if (siri_service_client_request(
+ pid,
+ port,
+ /* -1 = new pool */
+ (req == SERVICE_NEW_POOL) ? -1 : qp_pool.via.int64,
+ &uuid,
+ &qp_host,
+ &qp_username,
+ &qp_password,
+ &qp_dbname,
+ dbpath,
+ client,
+ err_msg))
+ {
+ siri_service_request_rollback(dbpath);
+ return CPROTO_ERR_SERVICE;
+ }
+ return CPROTO_DEFERRED;
+}
+
+/*
+ * Returns CPROTO_ACK_SERVICE_DATA when successful.
+ * In case of an error CPROTO_ERR_SERVICE will be returned and err_msg is set
+ */
+static cproto_server_t SERVICE_on_get_version(
+ qp_unpacker_t * qp_unpacker __attribute__((unused)),
+ qp_packer_t ** packaddr,
+ char * err_msg)
+{
+ qp_packer_t * packer = sirinet_packer_new(128);
+ if (packer != NULL)
+ {
+ if (!qp_add_type(packer, QP_ARRAY_OPEN) &&
+ !qp_add_string(packer, SIRIDB_VERSION))
+ {
+ *packaddr = packer;
+ return CPROTO_ACK_SERVICE_DATA;
+ }
+
+ /* error, free packer */
+ qp_packer_free(packer);
+ }
+ sprintf(err_msg, "memory allocation error");
+ return CPROTO_ERR_SERVICE;
+}
+
+static cproto_server_t SERVICE_on_get_accounts(
+ qp_unpacker_t * qp_unpacker __attribute__((unused)),
+ qp_packer_t ** packaddr,
+ char * err_msg)
+{
+ qp_packer_t * packer = sirinet_packer_new(128);
+
+ if (packer != NULL)
+ {
+ qp_add_type(packer, QP_ARRAY_OPEN);
+
+ if (!llist_walk(
+ siri.accounts,
+ (llist_cb) SERVICE_list_accounts,
+ packer))
+ {
+ *packaddr = packer;
+ return CPROTO_ACK_SERVICE_DATA;
+ }
+
+ /* error, free packer */
+ qp_packer_free(packer);
+ }
+ sprintf(err_msg, "memory allocation error");
+ return CPROTO_ERR_SERVICE;
+}
+
+static cproto_server_t SERVICE_on_get_databases(
+ qp_unpacker_t * qp_unpacker __attribute__((unused)),
+ qp_packer_t ** packaddr,
+ char * err_msg)
+{
+ qp_packer_t * packer = sirinet_packer_new(128);
+
+ if (packer != NULL)
+ {
+ qp_add_type(packer, QP_ARRAY_OPEN);
+
+ if (!llist_walk(
+ siri.siridb_list,
+ (llist_cb) SERVICE_list_databases,
+ packer))
+ {
+ *packaddr = packer;
+ return CPROTO_ACK_SERVICE_DATA;
+ }
+
+ /* error, free packer */
+ qp_packer_free(packer);
+ }
+ sprintf(err_msg, "memory allocation error");
+ return CPROTO_ERR_SERVICE;
+}
+
+void siri_service_request_rollback(const char * dbpath)
+{
+ size_t dbpath_len = strlen(dbpath);
+ char dbfn[dbpath_len + max_filename_sz];
+
+ sprintf(dbfn, "%s%s", dbpath, DB_CONF_FN);
+ unlink(dbfn);
+ sprintf(dbfn, "%s%s", dbpath, DB_DAT_FN);
+ unlink(dbfn);
+ sprintf(dbfn, "%s%s", dbpath, REINDEX_FN);
+ unlink(dbfn);
+ if (rmdir(dbpath))
+ {
+ log_error("Roll-back creating new database has failed.");
+ }
+}
+
+static int8_t SERVICE_time_precision(qp_obj_t * qp_time_precision)
+{
+ if (qp_time_precision->tp != QP_RAW)
+ {
+ return -1;
+ }
+ if (qp_time_precision->len == 1 && qp_time_precision->via.raw[0] == 's')
+ {
+ return 0;
+ }
+ else if (qp_time_precision->len == 2 && qp_time_precision->via.raw[1] == 's')
+ {
+ switch (qp_time_precision->via.raw[0])
+ {
+ case 'm': return 1;
+ case 'u': return 2;
+ case 'n': return 3;
+ }
+ }
+ return -1;
+}
+
+static int64_t SERVICE_duration(qp_obj_t * qp_duration, uint8_t time_precision)
+{
+ char * endptr;
+ long int val;
+
+ if (qp_duration->tp != QP_RAW || qp_duration->len < 2)
+ {
+ return -1;
+ }
+
+ val = strtol((const char *) qp_duration->via.raw, &endptr, 10);
+
+ if (val < 1 || val > 99 || endptr == (const char *) qp_duration->via.raw)
+ {
+ return -1;
+ }
+
+ if (endptr != (const char *) qp_duration->via.raw + (qp_duration->len - 1))
+ {
+ return -1;
+ }
+
+ switch (*endptr)
+ {
+ case 'h': return xmath_ipow(1000, time_precision) * val * 3600;
+ case 'd': return xmath_ipow(1000, time_precision) * val * 86400;
+ case 'w': return xmath_ipow(1000, time_precision) * val * 604800;
+ }
+
+ return -1;
+}
+
+static int SERVICE_list_databases(siridb_t * siridb, qp_packer_t * packer)
+{
+ return qp_add_string(packer, siridb->dbname);
+}
+
+static int SERVICE_find_database(siridb_t * siridb, qp_obj_t * dbname)
+{
+ return (
+ strlen(siridb->dbname) == dbname->len &&
+ strncmp(
+ siridb->dbname,
+ (const char *) dbname->via.raw,
+ dbname->len) == 0);
+}
+
+static int SERVICE_list_accounts(
+ siri_service_account_t * account,
+ qp_packer_t * packer)
+{
+ return qp_add_string(packer, account->account);
+}
+
+static void SERVICE_on_drop_database_cb(
+ vec_t * promises,
+ void * data __attribute__((unused)))
+{
+ log_debug("drop database has been send to all online servers");
+
+ if (promises != NULL)
+ {
+ size_t i;
+ sirinet_promise_t * promise;
+ for (i = 0; i < promises->len; i++)
+ {
+ promise = promises->data[i];
+ if (promise != NULL)
+ {
+ free(promise->data);
+ sirinet_promise_decref(promise);
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * siri.c - Root for SiriDB.
+ *
+ *
+ * Info siri->siridb_mutex:
+ *
+ * Main thread:
+ * siri->siridb_list : read (no lock) write (lock)
+ *
+ * Other threads:
+ * siri->siridb_list : read (lock) write (not allowed)
+ *
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <logger/logger.h>
+#include <qpack/qpack.h>
+#include <siri/async.h>
+#include <siri/buffersync.h>
+#include <siri/cfg/cfg.h>
+#include <siri/db/aggregate.h>
+#include <siri/db/buffer.h>
+#include <siri/db/groups.h>
+#include <siri/db/listener.h>
+#include <siri/db/pools.h>
+#include <siri/db/props.h>
+#include <siri/db/series.h>
+#include <siri/db/server.h>
+#include <siri/db/servers.h>
+#include <siri/db/users.h>
+#include <siri/api.h>
+#include <siri/err.h>
+#include <siri/health.h>
+#include <siri/help/help.h>
+#include <siri/net/bserver.h>
+#include <siri/net/clserver.h>
+#include <siri/net/pipe.h>
+#include <siri/net/stream.h>
+#include <siri/service/account.h>
+#include <siri/service/request.h>
+#include <siri/siri.h>
+#include <siri/version.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <xstr/xstr.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+#include <xpath/xpath.h>
+
+
+static void SIRI_signal_handler(uv_signal_t * req, int signum);
+static int SIRI_load_databases(void);
+static void SIRI_close_handlers(void);
+static void SIRI_walk_close_handlers(uv_handle_t * handle, void * arg);
+static void SIRI_destroy(void);
+static void SIRI_set_running_state(void);
+static void SIRI_set_closing_state(void);
+static void SIRI_try_close(uv_timer_t * handle);
+static void SIRI_walk_try_close(uv_handle_t * handle, int * num);
+
+#define WAIT_BETWEEN_CLOSE_ATTEMPTS 3000
+
+static uv_timer_t closing_timer;
+static int closing_attempts = 40; /* times 3 seconds is 2 minutes */
+
+#define N_SIGNALS 6
+static int signals[N_SIGNALS] = {
+ SIGHUP,
+ SIGINT,
+ SIGTERM,
+ SIGSEGV,
+ SIGABRT,
+ SIGPIPE};
+
+siri_t siri = {
+ .grammar=NULL,
+ .loop=NULL,
+ .siridb_list=NULL,
+ .fh=NULL,
+ .optimize=NULL,
+ .heartbeat=NULL,
+ .buffersync=NULL,
+ .cfg=NULL,
+ .args=NULL,
+ .status=SIRI_STATUS_LOADING,
+ .startup_time=0,
+ .accounts=NULL,
+ .dbname_regex=NULL,
+ .dbname_match_data=NULL,
+ .client=NULL};
+
+void siri_setup_logger(void)
+{
+ int n;
+ char lname[255];
+ size_t len = strlen(siri.args->log_level);
+
+#ifdef NDEBUG
+ /* force colors while debugging... */
+ if (siri.args->log_colorized)
+#endif
+ {
+ Logger.flags |= LOGGER_FLAG_COLORED;
+ }
+
+ for (n = 0; n < LOGGER_NUM_LEVELS; n++)
+ {
+ strcpy(lname, logger_level_name(n));
+ xstr_lower_case(lname);
+ if (strlen(lname) == len && strcmp(siri.args->log_level, lname) == 0)
+ {
+ logger_init(stdout, n);
+ return;
+ }
+ }
+
+ assert (0);
+ /* We should not get here since args should always
+ * contain a valid log level
+ */
+ logger_init(stdout, 0);
+}
+
+int make_database_directory(void)
+{
+ char tmppath[XPATH_MAX];
+ char * sysuser = getenv("USER");
+ char * homedir = getenv("HOME");
+ size_t len;
+
+ memset(tmppath, 0, XPATH_MAX);
+
+ if (*siri.cfg->db_path == '\0')
+ {
+ if (!homedir || !sysuser || strcmp(sysuser, "root") == 0)
+ {
+ strcpy(siri.cfg->db_path, "/var/lib/siridb/");
+ }
+ else
+ {
+ snprintf(siri.cfg->db_path, XPATH_MAX, "%s%s.siridb/",
+ homedir,
+ homedir[strlen(homedir)-1] == '/' ? "" : "/");
+ }
+ }
+
+ if (!xpath_is_dir(siri.cfg->db_path))
+ {
+ log_warning("Database directory not found, creating directory '%s'.",
+ siri.cfg->db_path);
+ if (mkdir(siri.cfg->db_path, 0700) == -1)
+ {
+ log_error("Cannot create directory '%s'.",
+ siri.cfg->db_path);
+ return -1;
+ }
+ }
+
+ if (realpath(siri.cfg->db_path, tmppath) == NULL)
+ {
+ log_warning(
+ "Could not resolve default database path: %s",
+ siri.cfg->db_path);
+ }
+ else
+ {
+ memcpy(siri.cfg->db_path, tmppath, sizeof(tmppath));
+ }
+
+ len = strlen(siri.cfg->db_path);
+
+ if (len >= XPATH_MAX - 2)
+ {
+ log_warning(
+ "Default database path exceeds %d characters, please "
+ "check your configuration file: %s",
+ XPATH_MAX - 3,
+ siri.args->config);
+ return -1;
+ }
+
+ /* add trailing slash (/) if its not already there */
+ if (siri.cfg->db_path[len - 1] != '/')
+ {
+ siri.cfg->db_path[len] = '/';
+ siri.cfg->db_path[len+1] = '\0';
+ }
+
+ return 0;
+}
+
+void set_max_open_files_limit(void)
+{
+ struct rlimit rlim;
+
+ if (siri.cfg->max_open_files < MIN_OPEN_FILES_LIMIT ||
+ siri.cfg->max_open_files > MAX_OPEN_FILES_LIMIT)
+ {
+ log_warning(
+ "Value max_open_files must be a value between %d and %d "
+ "but we found %d. Using default value instead: %d",
+ MIN_OPEN_FILES_LIMIT, MAX_OPEN_FILES_LIMIT,
+ siri.cfg->max_open_files, DEFAULT_OPEN_FILES_LIMIT);
+ siri.cfg->max_open_files = DEFAULT_OPEN_FILES_LIMIT;
+ }
+
+ getrlimit(RLIMIT_NOFILE, &rlim);
+
+ uint16_t min_limit = (uint16_t)
+ ((double) siri.cfg->max_open_files / RLIMIT_PERC_FOR_SHARDING) -1;
+
+ if (min_limit > (uint64_t) rlim.rlim_max)
+ {
+ siri.cfg->max_open_files =
+ (uint16_t) ((double) rlim.rlim_max * RLIMIT_PERC_FOR_SHARDING);
+ log_warning(
+ "We want to set a max-open-files value which "
+ "exceeds %d%% of the current hard limit.\n\nWe "
+ "will use %d as max_open_files for now.\n"
+ "Please increase the hard-limit using:\n"
+ "ulimit -Hn %d",
+ (uint8_t) (RLIMIT_PERC_FOR_SHARDING * 100),
+ siri.cfg->max_open_files,
+ min_limit);
+ min_limit = siri.cfg->max_open_files * 2;
+ }
+
+ if (min_limit > (uint64_t) rlim.rlim_cur)
+ {
+ rlim_t prev = rlim.rlim_cur;
+ log_info(
+ "Increasing soft-limit from %d to %d since we want "
+ "to use only %d%% from the soft-limit for shard files",
+ (uint64_t) rlim.rlim_cur,
+ min_limit,
+ (uint8_t) (RLIMIT_PERC_FOR_SHARDING * 100));
+ rlim.rlim_cur = min_limit;
+ if (setrlimit(RLIMIT_NOFILE, &rlim))
+ {
+ siri.cfg->max_open_files = (uint16_t) (prev / 2);
+ log_warning("Could not set the soft-limit to %d, "
+ "changing max open files to: %u",
+ min_limit, siri.cfg->max_open_files);
+ }
+ }
+}
+
+int siri_start(void)
+{
+ int rc;
+ struct timespec start;
+ struct timespec end;
+ uv_signal_t sig[N_SIGNALS];
+ int i;
+
+ /* get start time so we can calculate the startup_time */
+ clock_gettime(CLOCK_MONOTONIC, &start);
+
+ /* initialize listener (set enter and exit functions) */
+ siridb_init_listener();
+
+ /* initialize props (set props functions) */
+ siridb_init_props();
+
+ /* initialize aggregation */
+ siridb_init_aggregates();
+
+ /* load SiriDB grammar */
+ siri.grammar = compile_siri_grammar_grammar();
+
+ /* create store for SiriDB instances */
+ siri.siridb_list = llist_new();
+ if (siri.siridb_list == NULL)
+ {
+ return -1;
+ }
+
+ /* initialize file handler for shards */
+ siri.fh = siri_fh_new(siri.cfg->max_open_files);
+
+ /* initialize the default event loop */
+ siri.loop = malloc(sizeof(uv_loop_t));
+ if (siri.loop == NULL)
+ {
+ return -1;
+ }
+ uv_loop_init(siri.loop);
+
+ /* initialize the back-end-, client- server and load databases */
+ if ( (siri.cfg->http_status_port && (rc = siri_health_init())) ||
+ (siri.cfg->http_api_port && (rc = siri_api_init())) ||
+ (rc = siri_service_account_init(&siri)) ||
+ (rc = siri_service_request_init()) ||
+ (rc = sirinet_bserver_init(&siri)) ||
+ (rc = sirinet_clserver_init(&siri)) ||
+ (rc = SIRI_load_databases()))
+ {
+ SIRI_destroy();
+ free(siri.loop);
+ siri.loop = NULL;
+ return rc; /* something went wrong */
+ }
+
+ /* bind signals to the event loop */
+ for (i = 0; i < N_SIGNALS; i++)
+ {
+ uv_signal_init(siri.loop, &sig[i]);
+ uv_signal_start(&sig[i], SIRI_signal_handler, signals[i]);
+ }
+
+ /* initialize optimize task (bind siri.optimize) */
+ siri_optimize_init(&siri);
+
+ /* initialize heart-beat task (bind siri.heartbeat) */
+ siri_heartbeat_init(&siri);
+
+ /* initialize buffer-sync task (bind siri.buffersync) */
+ siri_buffersync_init(&siri);
+
+ /* initialize backup (bind siri.backup) */
+ if (siri_backup_init(&siri))
+ {
+ SIRI_destroy();
+ free(siri.loop);
+ siri.loop = NULL;
+ return -1;
+ }
+
+ /* update siridb status to running */
+ SIRI_set_running_state();
+
+ /* set startup time */
+ clock_gettime(CLOCK_MONOTONIC, &end);
+ siri.startup_time = end.tv_sec - start.tv_sec;
+
+ /* start the event loop */
+ uv_run(siri.loop, UV_RUN_DEFAULT);
+
+ /* quit, don't forget to run siri_free() (should be done in main) */
+ return 0;
+}
+
+void siri_free(void)
+{
+ if (siri.loop != NULL)
+ {
+ int rc;
+ rc = uv_loop_close(siri.loop);
+ if (rc) /* could be UV_EBUSY (-16) in case handlers are not closed */
+ {
+ log_error("Error occurred while closing the event loop: %d", rc);
+ }
+ }
+
+ /* first free the File Handler. (this will close all open shard files) */
+ siri_fh_free(siri.fh);
+
+ /* this will free each SiriDB database and the list */
+ llist_free_cb(siri.siridb_list, (llist_cb) siridb_decref_cb, NULL);
+
+ /* free siridb grammar */
+ cleri_grammar_free(siri.grammar);
+
+ /* free siridb service accounts */
+ siri_service_account_destroy(&siri);
+
+ /* free siridb service request */
+ siri_service_request_destroy();
+
+ /* free config */
+ siri_cfg_destroy(&siri);
+
+ /* free event loop */
+ free(siri.loop);
+}
+
+static int SIRI_load_databases(void)
+{
+ DIR * db_container_path;
+ struct dirent * dbpath;
+ char * buffer;
+
+ if (!xpath_is_dir(siri.cfg->db_path))
+ {
+ log_warning("Database directory not found, creating directory '%s'.",
+ siri.cfg->db_path);
+ if (mkdir(siri.cfg->db_path, 0700) == -1)
+ {
+ log_error("Cannot create directory '%s'.",
+ siri.cfg->db_path);
+ return -1;
+ }
+ }
+
+ if ((db_container_path = opendir(siri.cfg->db_path)) == NULL)
+ {
+ log_error("Cannot open database directory '%s'.",
+ siri.cfg->db_path);
+ return -1;
+ }
+
+ while((dbpath = readdir(db_container_path)) != NULL)
+ {
+ if ( strcmp(dbpath->d_name, ".") == 0 ||
+ strcmp(dbpath->d_name, "..") == 0 ||
+ strncmp(dbpath->d_name, "__", 2) == 0)
+ {
+ /* skip "." ".." and prefixed with double underscore directories */
+ continue;
+ }
+
+ if (asprintf(
+ &buffer,
+ "%s%s/",
+ siri.cfg->db_path,
+ dbpath->d_name) < 0)
+ {
+ /* allocation error occurred */
+ log_critical("Could not allocate space for database path");
+ continue;
+ }
+
+ if (!siridb_is_db_path(buffer))
+ {
+ /* this is not a SiriDB database directory, files are missing */
+ goto next;
+ }
+
+ if (siri.siridb_list->len == MAX_NUMBER_DB)
+ {
+ log_critical(
+ "Cannot load '%s' since no more than %d databases "
+ "are allowed on a single SiriDB process.",
+ dbpath->d_name,
+ MAX_NUMBER_DB);
+ goto next;
+ }
+
+ if (siridb_new(buffer, 0) == NULL)
+ {
+ log_error("Could not load '%s'.", dbpath->d_name);
+ }
+next:
+ free(buffer);
+ }
+ closedir(db_container_path);
+
+ return 0;
+}
+
+static void SIRI_destroy(void)
+{
+ log_warning("Closing SiriDB Server (version: %s)", SIRIDB_VERSION);
+ /* stop the event loop */
+ uv_stop(siri.loop);
+
+ /* use one iteration to close all open handlers */
+ SIRI_close_handlers();
+}
+
+static void SIRI_set_running_state(void)
+{
+ siri.status = SIRI_STATUS_RUNNING;
+
+ llist_node_t * db_node = siri.siridb_list->first;
+ while (db_node != NULL)
+ {
+ siridb_server_t * server = ((siridb_t *) db_node->data)->server;
+ server->flags |= SERVER_FLAG_RUNNING;
+ db_node = db_node->next;
+ }
+}
+
+static void SIRI_set_closing_state(void)
+{
+ siri.status = SIRI_STATUS_CLOSING;
+
+ llist_node_t * db_node = siri.siridb_list->first;
+ while (db_node != NULL)
+ {
+ siridb_t * siridb = (siridb_t *) db_node->data;
+ if (siridb->replicate != NULL)
+ {
+ siridb_replicate_close(siridb->replicate);
+ }
+ if (siridb->reindex != NULL && siridb->reindex->timer != NULL)
+ {
+ siridb_reindex_close(siridb->reindex);
+ }
+ if (siridb->groups != NULL)
+ {
+ siridb_groups_destroy(siridb->groups);
+ }
+ siridb->server->flags &= ~SERVER_FLAG_RUNNING;
+ siridb_servers_send_flags(siridb->servers);
+
+ db_node = db_node->next;
+ }
+}
+
+static void SIRI_walk_try_close(uv_handle_t * handle, int * num)
+{
+ if (handle->type == UV_ASYNC || handle->type == UV_TIMER)
+ {
+ (*num)++;
+ }
+}
+
+static void SIRI_try_close(uv_timer_t * handle)
+{
+ int num = -1; /* minus one because we should not include 'this' timer */
+
+ uv_walk(siri.loop, (uv_walk_cb) SIRI_walk_try_close, &num);
+
+ if (!--closing_attempts && num)
+ {
+ log_error("SiriDB will close but still had %d task(s) running.", num);
+ /*
+ * We usually assume all async tasks and timers will finish 'normal'
+ * and take care of destroying the handle. Since now we will loop and
+ * force all handlers to close we must be able to act on this behavior.
+ * Therefore we set siri_err which can be checked.
+ */
+ siri_err = ERR_CLOSE_TIMEOUT_REACHED;
+ num = 0;
+ }
+
+ if (!num)
+ {
+ /* close this timer */
+ uv_timer_stop(handle);
+ uv_close((uv_handle_t *) handle, NULL);
+
+ /* stop SiriDB */
+ SIRI_destroy();
+ }
+ else
+ {
+ log_info(
+ "SiriDB is closing but is waiting for %d task(s) to finish...",
+ num);
+ }
+}
+
+static void SIRI_signal_handler(
+ uv_signal_t * req __attribute__((unused)),
+ int signum)
+{
+ if (signum == SIGPIPE)
+ {
+ log_warning(
+ "Signal (%d) received, probably a connection was lost",
+ signum);
+ return;
+ }
+
+ if (siri.status == SIRI_STATUS_CLOSING)
+ {
+ log_error(
+ "Receive a second signal (%d), stop SiriDB immediately!",
+ signum);
+ /* set siri_err, see ERR_CLOSE_TIMEOUT_REACHED for the reason why */
+ siri_err = ERR_CLOSE_ENFORCED;
+ SIRI_destroy();
+ }
+ else
+ {
+ /* stop optimize task */
+ siri_optimize_stop();
+
+ /* stop heart-beat task */
+ siri_heartbeat_stop(&siri);
+
+ /* stop buffer-sync task */
+ siri_buffersync_stop(&siri);
+
+ /* destroy backup (mode) task */
+ siri_backup_destroy(&siri);
+
+ /* mark SiriDB as closing and remove ONLINE flag from servers. */
+ SIRI_set_closing_state();
+
+ if (signum == SIGINT || signum == SIGTERM || signum == SIGHUP)
+ {
+ log_warning("Asked SiriDB Server to stop (%d)", signum);
+ }
+ else
+ {
+ log_critical("Signal (%d) received, stop SiriDB!",
+ signum);
+
+ /* set siri_err, see ERR_CLOSE_TIMEOUT_REACHED for the reason why */
+ if (!siri_err)
+ {
+ siri_err = signum;
+ }
+ }
+ /*
+ * Try to finish open tasks
+ */
+ uv_timer_init(siri.loop, &closing_timer);
+ uv_timer_start(
+ &closing_timer,
+ SIRI_try_close,
+ 0,
+ WAIT_BETWEEN_CLOSE_ATTEMPTS);
+ }
+}
+
+static void SIRI_walk_close_handlers(
+ uv_handle_t * handle,
+ void * arg __attribute__((unused)))
+{
+ if (uv_is_closing(handle))
+ {
+ return;
+ }
+
+ switch (handle->type)
+ {
+ case UV_SIGNAL:
+ /* this is where we cleanup the signal handlers */
+ uv_close(handle, NULL);
+ break;
+
+ case UV_TCP:
+ case UV_NAMED_PIPE:
+ {
+ if (handle->data == NULL || siridb_tee_is_handle(handle))
+ {
+ uv_close(handle, NULL);
+ }
+ else if (siri_health_is_handle(handle))
+ {
+ siri_health_close((siri_health_request_t *) handle->data);
+ }
+ else
+ {
+ sirinet_stream_decref((sirinet_stream_t *) handle->data);
+ }
+ }
+ break;
+
+ case UV_TIMER:
+ /* we do not expect any timer object since they should all be closed
+ * (or at least closing) at this point.
+ */
+#ifndef NDEBUG
+ LOGC( "Found a non closing Timer, all timers should "
+ "be stopped at this point.");
+#endif
+ uv_timer_stop((uv_timer_t *) handle);
+ uv_close(handle, NULL);
+ break;
+
+ case UV_ASYNC:
+#ifndef NDEBUG
+ LOGC( "An async task is only expected to be found in case "
+ "not all tasks were closed within the timeout limit, "
+ "or when a critical signal error is raised.");
+#endif
+ uv_close(handle, siri_async_close);
+ break;
+
+ default:
+
+#ifndef NDEBUG
+ LOGC("Oh oh, we might need to implement type %d", handle->type);
+ assert(0);
+#endif
+
+ break;
+ }
+}
+
+static void SIRI_close_handlers(void)
+{
+ /* close open handlers */
+ uv_walk(siri.loop, SIRI_walk_close_handlers, NULL);
+
+ /* run the loop once more so call-backs on uv_close() can run */
+ uv_run(siri.loop, UV_RUN_NOWAIT);
+}
--- /dev/null
+/*
+ * version.c - SiriDB version info.
+ */
+#include <assert.h>
+#include <siri/version.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int siri_version_cmp(const char * version_a, const char * version_b)
+{
+ long int a, b;
+ int i = 3;
+
+ char * str_a = (char *) version_a;
+ char * str_b = (char *) version_b;
+
+ while (i--)
+ {
+ a = strtol(str_a, &str_a, 10);
+ b = strtol(str_b, &str_b, 10);
+
+ if (a != b)
+ {
+ return a - b;
+ }
+ else if (!*str_a || !*str_b)
+ {
+ return 0;
+ }
+ str_a++;
+ str_b++;
+ }
+
+ return 0;
+}
--- /dev/null
+/*
+ * timeit.c - Timeit.
+ */
+#include <timeit/timeit.h>
+#include <time.h>
+
+/*
+ * Returns time past in seconds since given start time.
+ */
+double timeit_get(struct timespec * start)
+{
+ struct timespec end;
+
+ clock_gettime(CLOCK_MONOTONIC, &end);
+
+ return (end.tv_sec - start->tv_sec) +
+ (end.tv_nsec - start->tv_nsec) / 1000000000.0f;
+}
+
--- /dev/null
+/*
+ * vec.c - Vector List.
+ */
+#include <logger/logger.h>
+#include <vec/vec.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define VEC_MAX_SZ 512
+
+/*
+ * Returns NULL in case an error has occurred.
+ *
+ * In case the size is unknown, VEC_DEFAULT_SIZE is recommended since in this
+ * case we can do a re-allocation with multiples of 64K.
+ */
+vec_t * vec_new(size_t size)
+{
+ /* sizeof(vec_t) is 16 bytes, only for len and size and data[] is
+ * excluded.
+ */
+ vec_t * vec = malloc(sizeof(vec_t) + sizeof(void *) * size);
+
+ if (vec == NULL)
+ {
+ return NULL;
+ }
+ vec->size = size;
+ vec->len = 0;
+ return vec;
+}
+
+void vec_destroy(vec_t * vec, vec_destroy_cb cb)
+{
+ if (vec != NULL && cb)
+ {
+ size_t i;
+ for (i = 0; i < vec->len; ++i)
+ {
+ cb(vec->data[i]);
+ }
+ }
+ free(vec);
+}
+
+/*
+ * Returns NULL in case an error has occurred.
+ */
+vec_t * vec_copy(vec_t * source)
+{
+ size_t size = sizeof(vec_t) + sizeof(void *) * source->size;
+ vec_t * vec = malloc(size);
+ if (vec == NULL)
+ {
+ return NULL;
+ }
+ memcpy(vec, source, size);
+ return vec;
+}
+
+/*
+ * Returns 0 if successful or -1 in case of an error.
+ * (in case of an error the list is unchanged)
+ */
+int vec_append_safe(vec_t ** vec, void * data)
+{
+ if ((*vec)->len == (*vec)->size)
+ {
+ vec_t * tmp;
+
+ size_t sz = (*vec)->size;
+
+ /* double the size when > 0 and < VEC_MAX_SZ */
+ (*vec)->size = (sz >= VEC_DEFAULT_SIZE) ?
+ (sz <= VEC_MAX_SZ) ?
+ sz * 2 : sz + VEC_MAX_SZ : VEC_DEFAULT_SIZE;
+
+ tmp = realloc(*vec, sizeof(vec_t) + sizeof(void *) * (*vec)->size);
+ if (tmp == NULL)
+ {
+ /* an error has occurred */
+ (*vec)->size = sz;
+ return -1;
+ }
+
+ /* overwrite the original value with the new one */
+ *vec = tmp;
+ }
+
+ vec_append((*vec), data);
+
+ return 0;
+}
+
+/*
+ * Compact memory used for the vec object when the list has more than
+ * VEC_DEFAULT_SIZE free space. After the compact the list has
+ * a size which is VEC_DEFAULT_SIZE greater than its length.
+ */
+void vec_compact(vec_t ** vec)
+{
+ size_t sz = (*vec)->size;
+
+ if (sz - (*vec)->len > VEC_DEFAULT_SIZE)
+ {
+ vec_t * tmp;
+
+ (*vec)->size = (*vec)->len + VEC_DEFAULT_SIZE;
+
+ tmp = realloc(*vec, sizeof(vec_t) + sizeof(void *) * (*vec)->size);
+
+ if (tmp == NULL)
+ {
+ /* an error has occurred; log and restore size */
+ log_error("Error has occurred while re-allocating less space");
+ (*vec)->size = sz;
+ }
+ else
+ {
+ /* overwrite the original value with the new one */
+ *vec = tmp;
+ }
+ }
+}
--- /dev/null
+/*
+ * xmath.c - Extra math functions functions used by SiriDB.
+ */
+#include <xmath/xmath.h>
+#include <stdarg.h>
+
+/*
+ * Got this from : (Elias Yarrkov)
+ * http://stackoverflow.com/questions/101439/the-most-efficient-way-to-
+ * implement-an-integer-based-power-function-powint-int
+ */
+uint32_t xmath_ipow(int base, int exp)
+{
+ uint32_t result = 1;
+ while (exp)
+ {
+ if (exp & 1)
+ {
+ result *= base;
+ }
+ exp >>= 1;
+ base *= base;
+ }
+
+ return result;
+}
+
+size_t xmath_max_size(size_t n, ...)
+{
+ size_t t, m = 0;
+ va_list args;
+ va_start(args, n);
+ while (n--)
+ {
+ t = va_arg(args, size_t);
+ m = (t > m) ? t : m;
+ }
+ va_end(args);
+ return m;
+}
--- /dev/null
+/*
+ * xpath.c - Path and file tools.
+ */
+#include <limits.h>
+#include <logger/logger.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <xpath/xpath.h>
+
+/*
+ * Test if file exist using the effective user.
+ */
+int xpath_file_exist(const char * fn)
+{
+ FILE * fp;
+ fp = fopen(fn, "r");
+ if (fp == NULL)
+ {
+ return 0;
+ }
+ fclose(fp);
+ return 1;
+}
+
+/*
+ * Test if a path is a directory
+ */
+int xpath_is_dir(const char * path)
+{
+ struct stat st;
+ memset(&st, 0, sizeof(struct stat));
+ stat(path, &st);
+ return S_ISDIR(st.st_mode);
+}
+
+/*
+ * Returns the length of the content for a file and set buffer with the file
+ * content. Note that malloc is used to allocate memory for the buffer.
+ *
+ * In case of an error -1 is returned and buffer will be set to NULL.
+ */
+ssize_t xpath_get_content(char ** buffer, const char * fn)
+{
+ ssize_t size = 0;
+ *buffer = NULL;
+
+ FILE * fp = fopen(fn, "r");
+ if (fp == NULL)
+ {
+ log_critical("Cannot open file: '%s'", fn);
+ return -1;
+ }
+
+ if (fseeko(fp, 0, SEEK_END) == 0 &&
+ (size = ftello(fp)) != -1 &&
+ fseeko(fp, 0, SEEK_SET) == 0)
+ {
+ *buffer = malloc(size);
+ if (*buffer != NULL)
+ {
+ if (fread(*buffer, size, 1, fp) != 1)
+ {
+ log_critical("Could not get full content from '%s'", fn);
+ free(*buffer);
+ *buffer = NULL;
+ }
+ }
+ }
+
+ fclose(fp);
+ return (*buffer == NULL) ? -1 : size;
+}
+
+/*
+ * Get the current executable path.
+ * (path should at least have size XPATH_MAX)
+ *
+ * Returns 0 if successful or -1 in case of an error.
+ * (this functions writes logging in case of errors)
+ */
+int xpath_get_exec_path(char * path)
+{
+ char* path_end;
+
+ if (readlink("/proc/self/exe", path, XPATH_MAX) == -1)
+ {
+ log_critical("Cannot read executable path");
+ return -1;
+ }
+
+ /* find last / in path */
+ path_end = strrchr(path, '/');
+
+ if (path_end == NULL)
+ {
+ log_critical("Cannot find / in executable path");
+ return -1;
+ }
+
+ *(++path_end) = '\0';
+
+ return 0;
+}
+
+int xpath_rmdir(const char * path)
+{
+ DIR * d = opendir(path);
+ if (!d)
+ return -1;
+
+ size_t bufsz = 0, path_len = strlen(path);
+ const char * slash = (path[path_len - 1] == '/') ? "" : "/";
+ struct dirent * p;
+ char * buf = NULL;
+
+ while ((p = readdir(d)))
+ {
+ size_t len;
+
+ /* Skip the names "." and ".." as we don't want to recurse on them. */
+ if (!strcmp(p->d_name, ".") || !strcmp(p->d_name, ".."))
+ continue;
+
+ len = path_len + strlen(p->d_name) + 2;
+ if (len > bufsz)
+ {
+ bufsz = len;
+ char * tmp = realloc(buf, bufsz);
+ if (!tmp) goto stop;
+ buf = tmp;
+ }
+
+ sprintf(buf, "%s%s%s", path, slash, p->d_name);
+
+ if (xpath_is_dir(buf) ? xpath_rmdir(buf) : unlink(buf))
+ goto stop;
+ }
+
+stop:
+ free(buf);
+ closedir(d);
+
+ return rmdir(path);
+}
--- /dev/null
+/*
+ * xstr.c - Extra String functions used by SiriDB.
+ */
+#include <assert.h>
+#include <ctype.h>
+#include <inttypes.h>
+#include <logger/logger.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+void xstr_lower_case(char * sptr)
+{
+ for (; *sptr != '\0'; sptr++)
+ {
+ *sptr = tolower( (unsigned char) * sptr);
+ }
+}
+
+void xstr_upper_case(char * sptr)
+{
+ for (; *sptr != '\0'; sptr++)
+ {
+ *sptr = toupper( (unsigned char) * sptr);
+ }
+}
+
+void xstr_replace_char(char * sptr, char orig, char repl)
+{
+ for (; *sptr != '\0'; sptr++)
+ {
+ if (*sptr == orig)
+ {
+ *sptr = repl;
+ }
+ }
+}
+
+/*
+ * Replace all occurrences of 'o' with 'r' in 'str'. We restrict the new size
+ * to 'n'.
+ *
+ * Returns 0 if successful or -1 if the replaced string does not fit. In this
+ * case the original string is untouched. The new string is terminated.
+ */
+int xstr_replace_str(char * str, char * o, char * r, size_t n)
+{
+ char buffer[n];
+ char * pos, * s;
+ size_t l, size = 0, olen = strlen(o), rlen = strlen(r);
+
+ for (s = str; (pos = strstr(s, o)) != NULL; s = pos + olen)
+ {
+ l = pos - s;
+
+ if (size + l + rlen >= n)
+ {
+ return -1;
+ }
+
+ memcpy(buffer + size, s, l);
+ size += l;
+
+ memcpy(buffer + size, r, rlen);
+ size += rlen;
+
+ }
+
+ if (s != str)
+ {
+ memcpy(str, buffer, size);
+ str[size] = '\0';
+ }
+
+ return 0;
+}
+
+/*
+ * Split and then join a given string.
+ *
+ * For example:
+ * string: " this is a test "
+ * split: ' ' and join with '_'
+ * result: "this_is_a_test"
+ */
+void xstr_split_join(char * pt, char split_chr, char join_chr)
+{
+ int join = -1;
+ char * dest = pt;
+
+ for (; *pt != '\0'; pt++)
+ {
+ if (*pt != split_chr)
+ {
+ if (join > 0)
+ {
+ *dest = join_chr;
+ dest++;
+ }
+ join = 0;
+ *dest = *pt;
+ dest++;
+ }
+ else if (!join)
+ {
+ join = 1;
+ }
+ }
+
+ *dest = '\0';
+}
+
+void xstr_trim(char ** str, char chr)
+{
+ /*
+ * trim: when chr is 0 we will trim whitespace,
+ * otherwise only the given char.
+ */
+ char * end;
+
+ /* trim leading chars */
+ while ((chr && **str == chr) || (!chr && isspace(**str)))
+ {
+ (*str)++;
+ }
+
+ /* check all chars? */
+ if(**str == 0)
+ {
+ return;
+ }
+
+ /* trim trailing chars */
+ end = *str + strlen(*str) - 1;
+ while (end > *str &&
+ ((chr && *end == chr) || (!chr && isspace(*end))))
+ {
+ end--;
+ }
+
+ /* write new null terminator */
+ *(end + 1) = 0;
+}
+
+/*
+ * returns true or false
+ */
+bool xstr_is_empty(const char * str)
+{
+ const char * test = str;
+ for (; *test; test++)
+ {
+ if (!isspace(*test))
+ {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool xstr_is_int(const char * str)
+{
+ /* Handle negative numbers. */
+ if (*str == '-')
+ {
+ ++str;
+ }
+
+ /* Handle empty string or just "-". */
+ if (!*str)
+ {
+ return false;
+ }
+
+ /* Check for non-digit chars in the rest of the string. */
+ while (*str)
+ {
+ if (!isdigit(*str))
+ {
+ return false;
+ }
+ else
+ {
+ ++str;
+ }
+ }
+
+ return true;
+}
+
+bool xstr_is_float(const char * str)
+{
+ /* Handle negative numbers. */
+ if (*str == '-' || *str == '+')
+ {
+ ++str;
+ }
+
+ size_t dots = 0;
+
+ /* Handle empty string or just "-". */
+ if (!*str)
+ {
+ return false;
+ }
+
+ /* Check for non-digit chars in the rest of the string. */
+ while (*str)
+ {
+ if (*str == '.')
+ {
+ ++dots;
+ }
+ else if (!isdigit(*str))
+ {
+ return false;
+ }
+
+ ++str;
+ }
+
+ return dots == 1;
+}
+
+bool xstr_is_graph(const char * str)
+{
+ for (; *str; str++)
+ {
+ if (!isgraph(*str))
+ {
+ return false;
+ }
+ }
+ return true;
+}
+
+/*
+ * This function is used to extra a SiriDB string. These strings start
+ * with " or with ' and we should replace double this character in the
+ * string with a single one.
+ *
+ * 'dest' string will be terminated and the return value is the new
+ * length of 'dest'.
+ */
+size_t xstr_extract_string(char * dest, const char * source, size_t len)
+{
+ size_t i = 0;
+
+ /* take the first character, this is " or ' */
+ char chr = *source;
+
+ /* we need to loop till len-2 so take 2 */
+ for (len -= 2; i < len; i++)
+ {
+ source++;
+ if (*source == chr)
+ {
+ /* this is the character, skip one and take the next */
+ source++;
+ len--;
+ }
+ dest[i] = *source;
+ }
+
+ /* set final 0 */
+ dest[i] = 0;
+
+ return i;
+}
+
+/*
+ * Returns a string to double.
+ * No error checking is done, we make the following assumptions:
+ * - len > 0
+ * - string is allowed to have one dot (.) at most but not required
+ * - string can start with a plus (+) or minus (-) sign.
+ */
+double xstr_to_double(const char * str)
+{
+ double d;
+ double negative = 0;
+
+ switch (*str)
+ {
+ case '-':
+ negative = -1.0;
+ ++str;
+ break;
+ case '+':
+ ++str;
+ break;
+ }
+
+ if (*str == 'i')
+ return negative ? negative * INFINITY : INFINITY;
+
+ if (*str == 'n')
+ return NAN;
+
+ if (errno == ERANGE)
+ errno = 0;
+
+ d = strtod(str, NULL);
+
+ if (errno == ERANGE)
+ {
+ assert (d == HUGE_VAL || d == -HUGE_VAL);
+
+ d = d == HUGE_VAL ? INFINITY : -INFINITY;
+ errno = 0;
+ }
+
+ return negative ? negative * d : d;
+}
+
+/*
+ * Returns a string to uint64_t.
+ * No error checking is done, we make the following assumptions:
+ * - len > 0
+ * - string can only contain characters [0..9] and no signs
+ */
+uint64_t xstr_to_uint64(const char * src, size_t len)
+{
+ char * pt = (char *) src;
+
+ uint64_t i = *pt - '0';
+
+ while (--len && isdigit(*(++pt)))
+ {
+ i = 10 * i + *pt - '0';
+ }
+
+ return i;
+}
+
+/*
+ * Returns a string duplicate like strdup() and set the strlen() to n;
+ */
+char * xstr_dup(const char * src, size_t * n)
+{
+ *n = strlen(src);
+ char * nstr = malloc(*n + 1);
+ if (nstr)
+ {
+ memcpy(nstr, src, *n + 1);
+ }
+ return nstr;
+}
--- /dev/null
+#ifndef SIRIDB_TEST_H_
+#define SIRIDB_TEST_H_
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sys/time.h>
+
+#define TEST_OK 0
+#define TEST_FAILED 1
+#define TEST_MSG_OK "....\x1B[32mOK\x1B[0m"
+#define TEST_MSG_FAILED "\x1B[31mFAILED\x1B[0m"
+
+static struct timeval start;
+static struct timeval end;
+
+static int status = TEST_OK;
+static int count = 0;
+
+const char * padding =
+ ".............................."
+ "..............................";
+
+static void test_start(char * test_name)
+{
+ count = 0;
+ int padlen = 60 - strlen(test_name);
+ printf("Testing %s%*.*s", test_name, padlen, padlen, padding);
+ gettimeofday(&start, 0);
+}
+
+static int test_end(void)
+{
+ gettimeofday(&end, 0);
+ float t = (end.tv_sec - start.tv_sec) * 1000.0f +
+ (end.tv_usec - start.tv_usec) / 1000.0f;
+
+ printf("%s (%.3f ms)\n",
+ (status == TEST_OK) ? TEST_MSG_OK : TEST_MSG_FAILED,
+ t);
+
+ return status;
+}
+
+#define _assert(e) (void)((e)?count++:(status = TEST_FAILED) && printf("\n\x1B[33mAssertion failed (%s:%d):\x1B[0m %s\n\n", __FILE__, __LINE__, #e))
+
+#endif /* SIRIDB_TEST_H_ */
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+RET=0
+echo -n "Test using valgrind for memory errors and leaks: "
+if [[ "$NOMEMTEST" -ne "1" ]] && hash valgrind 2>/dev/null; then
+ NOMEMTEST=0;
+ echo -e "\x1B[32menabled\x1B[0m";
+else
+ NOMEMTEST=1;
+ echo -e "\x1B[33mdisabled\x1B[0m";
+fi
+
+if [[ "$OSTYPE" == "darwin" ]]; then
+ LCRYPT=
+else
+ LCRYPT=-lcrypt
+fi
+
+run () {
+ if [ ! -f $1/sources ]; then
+ return;
+ fi
+ C_SRC=$(cat $1/sources)
+
+ SOURCE=$1/$1.c
+ OUT=$1.out
+ rm "$OUT" 2> /dev/null
+
+ gcc -I"../include" -O0 -g3 -Wall -Wextra -Winline -std=gnu99 $SOURCE $C_SRC -lm -lpcre2-8 -lcleri -luuid -luv -lyajl $LCRYPT -o "$OUT"
+ if [[ "$NOMEMTEST" -ne "1" ]]; then
+ valgrind --tool=memcheck --error-exitcode=1 --leak-check=full -q ./$OUT
+ else
+ ./$OUT
+ fi
+ rc=$?; if [[ $rc != 0 ]]; then RET=$((RET+1)); fi
+ rm "$OUT" 2> /dev/null
+ rm -r "$OUT.dSYM" 2> /dev/null
+}
+
+if [ $# -eq 0 ]; then
+ for d in test_*/ ; do
+ run "${d%?}"
+ done
+else
+ name=`echo $1 | sed 's/\(test_\)\?\(.*\?\)$/\2/g' | sed 's/\(.*\)\/$/\1/g'`
+ run "test_$name"
+fi
+
+exit $RET
\ No newline at end of file
--- /dev/null
+../src/siri/db/access.c
--- /dev/null
+#include "../test.h"
+#include <siri/db/access.h>
+
+
+int main()
+{
+ test_start("access");
+
+ char buffer[SIRIDB_ACCESS_STR_MAX];
+ uint32_t access_bit = 0;
+
+ siridb_access_to_str(buffer, access_bit);
+ _assert (strcmp(buffer, "no access") == 0);
+
+ access_bit |= SIRIDB_ACCESS_SHOW;
+ siridb_access_to_str(buffer, access_bit);
+ _assert (strcmp(buffer, "show") == 0);
+
+ access_bit |= SIRIDB_ACCESS_SELECT;
+ siridb_access_to_str(buffer, access_bit);
+ _assert (strcmp(buffer, "select and show") == 0);
+
+ access_bit |= SIRIDB_ACCESS_LIST;
+ siridb_access_to_str(buffer, access_bit);
+ _assert (strcmp(buffer, "list, select and show") == 0);
+
+ access_bit |= SIRIDB_ACCESS_PROFILE_WRITE;
+ siridb_access_to_str(buffer, access_bit);
+ _assert (strcmp(buffer, "write") == 0);
+
+ access_bit &= ~SIRIDB_ACCESS_INSERT;
+ siridb_access_to_str(buffer, access_bit);
+ _assert (strcmp(buffer, "read and create") == 0);
+
+ _assert (siridb_access_from_strn("read", 4) == (SIRIDB_ACCESS_PROFILE_READ));
+ _assert (siridb_access_from_strn("list", 4) == SIRIDB_ACCESS_LIST);
+
+ return test_end();
+}
\ No newline at end of file
--- /dev/null
+../src/siri/db/aggregate.c
+../src/siri/db/points.c
+../src/siri/db/variance.c
+../src/siri/db/median.c
+../src/siri/db/re.c
+../src/siri/err.c
+../src/qpack/qpack.c
+../src/vec/vec.c
+../src/cexpr/cexpr.c
+../src/xstr/xstr.c
+../src/logger/logger.c
--- /dev/null
+#include <math.h>
+#include "../test.h"
+#include <siri/db/points.h>
+#include <siri/db/aggregate.h>
+
+
+#define SIRIDB_MAX_SIZE_ERR_MSG 1024
+
+static siridb_aggr_t aggr;
+static char err_msg[SIRIDB_MAX_SIZE_ERR_MSG];
+
+
+static siridb_points_t * prepare_points(void)
+{
+ siridb_points_t * points = siridb_points_new(10, TP_INT);
+ uint64_t timestamps[10] = {3, 6, 7, 10, 11, 13, 14, 15, 25, 27};
+ int64_t values[10] = {1, 3, 0, 2, 4, 8, 3, 5, 6, 3};
+ qp_via_t val;
+ unsigned int i;
+
+ siridb_init_aggregates();
+
+ for (i = 0; i < 10; i++)
+ {
+ val.int64 = values[i];
+ siridb_points_add_point(points, ×tamps[i], &val);
+ }
+
+ return points;
+}
+
+static int test_count(void)
+{
+ test_start("aggr (count)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_COUNT;
+ aggr.group_by = 6;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 4);
+ _assert (aggrp->tp == TP_INT);
+ _assert (aggrp->data->ts == 6 && aggrp->data->val.int64 == 2);
+ _assert ((aggrp->data + 3)->ts == 30 &&
+ (aggrp->data + 3)->val.int64 == 2);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_first(void)
+{
+ test_start("aggr (first)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_FIRST;
+ aggr.group_by = 5;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 5);
+ _assert (aggrp->tp == TP_INT);
+ _assert (aggrp->data->ts == 5 && aggrp->data->val.int64 == 1);
+ _assert ((aggrp->data + 2)->ts == 15 &&
+ (aggrp->data + 2)->val.int64 == 4);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_last(void)
+{
+ test_start("aggr (last)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_LAST;
+ aggr.group_by = 5;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 5);
+ _assert (aggrp->tp == TP_INT);
+ _assert (aggrp->data->ts == 5 && aggrp->data->val.int64 == 1);
+ _assert ((aggrp->data + 2)->ts == 15 &&
+ (aggrp->data + 2)->val.int64 == 5);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_max(void)
+{
+ test_start("aggr (max)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_MAX;
+ aggr.group_by = 10;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 3);
+ _assert (aggrp->tp == TP_INT);
+ _assert (aggrp->data->ts == 10 && aggrp->data->val.int64 == 3);
+ _assert ((aggrp->data + 2)->ts == 30 &&
+ (aggrp->data + 2)->val.int64 == 6);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+
+static int test_mean(void)
+{
+ test_start("aggr (mean)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_MEAN;
+ aggr.group_by = 4;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 5);
+ _assert (aggrp->tp == TP_DOUBLE);
+ _assert (aggrp->data->ts == 4 && aggrp->data->val.real == 1.0);
+ _assert ((aggrp->data + 4)->ts == 28 &&
+ (aggrp->data + 4)->val.real == 4.5);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_median(void)
+{
+ test_start("aggr (median)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_MEDIAN;
+ aggr.group_by = 7;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 4);
+ _assert (aggrp->tp == TP_DOUBLE);
+ _assert (aggrp->data->ts == 7 && aggrp->data->val.real == 1.0);
+ _assert ((aggrp->data + 1)->ts == 14 &&
+ (aggrp->data + 1)->val.real == 3.5);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_median_high(void)
+{
+ test_start("aggr (median_high)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_MEDIAN_HIGH;
+ aggr.group_by = 7;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 4);
+ _assert (aggrp->tp == TP_INT);
+ _assert (aggrp->data->ts == 7 && aggrp->data->val.int64 == 1);
+ _assert ((aggrp->data + 1)->ts == 14 &&
+ (aggrp->data + 1)->val.int64 == 4);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_median_low(void)
+{
+ test_start("aggr (median_low)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_MEDIAN_LOW;
+ aggr.group_by = 7;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 4);
+ _assert (aggrp->tp == TP_INT);
+ _assert (aggrp->data->ts == 7 && aggrp->data->val.int64 == 1);
+ _assert ((aggrp->data + 1)->ts == 14 &&
+ (aggrp->data + 1)->val.int64 == 3);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_min(void)
+{
+ test_start("aggr (min)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_MIN;
+ aggr.group_by = 2;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 9);
+ _assert (aggrp->tp == TP_INT);
+ _assert (aggrp->data->ts == 4 && aggrp->data->val.int64 == 1);
+ _assert ((aggrp->data + 5)->ts == 14 &&
+ (aggrp->data + 5)->val.int64 == 3);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_pvariance(void)
+{
+ test_start("aggr (pvariance)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_PVARIANCE;
+ aggr.group_by = 5;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 5);
+ _assert (aggrp->tp == TP_DOUBLE);
+ _assert (aggrp->data->ts == 5 && aggrp->data->val.real == 0.0);
+ _assert ((aggrp->data + 2)->ts == 15 &&
+ (aggrp->data + 2)->val.real == 3.5);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_stddev(void)
+{
+ test_start("aggr (stddev)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_STDDEV;
+ aggr.group_by = 6;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 4);
+ _assert (aggrp->tp == TP_DOUBLE);
+ _assert (aggrp->data->ts == 6 && aggrp->data->val.real == sqrt(2.0));
+ _assert ((aggrp->data + 1)->ts == 12 &&
+ (aggrp->data + 1)->val.real == 2.0);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_sum(void)
+{
+ test_start("aggr (sum)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_SUM;
+ aggr.group_by = 5;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 5);
+ _assert (aggrp->tp == TP_INT);
+ _assert (aggrp->data->ts == 5 && aggrp->data->val.int64 == 1);
+ _assert ((aggrp->data + 2)->ts == 15 &&
+ (aggrp->data + 2)->val.int64 == 20);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+static int test_variance(void)
+{
+ test_start("aggr (variance)");
+
+ siridb_points_t * aggrp, * points = prepare_points();
+
+ aggr.gid = CLERI_GID_F_VARIANCE;
+ aggr.group_by = 6;
+ aggr.limit = 0;
+ aggr.offset = 0;
+
+ aggrp = siridb_aggregate_run(points, &aggr, err_msg);
+
+ _assert (aggrp != NULL);
+ _assert (aggrp->len == 4);
+ _assert (aggrp->tp == TP_DOUBLE);
+ _assert (aggrp->data->ts == 6 && aggrp->data->val.real == 2.0);
+ _assert ((aggrp->data + 1)->ts == 12 &&
+ (aggrp->data + 1)->val.real == 4.0);
+
+ siridb_points_free(aggrp);
+ siridb_points_free(points);
+
+ return test_end();
+}
+
+int main()
+{
+ return (
+ test_count() ||
+ test_first() ||
+ test_last() ||
+ test_max() ||
+ test_mean() ||
+ test_median() ||
+ test_median_high() ||
+ test_median_low() ||
+ test_min() ||
+ test_pvariance() ||
+ test_stddev() ||
+ test_sum() ||
+ test_variance() ||
+ 0
+ );
+}
--- /dev/null
+../src/ctree/ctree.c
+../src/logger/logger.c
--- /dev/null
+#include "../test.h"
+#include <ctree/ctree.h>
+
+static const unsigned int num_entries = 14;
+static char * entries[] = {
+ "Zero",
+ "First entry",
+ "Second entry",
+ "Third entry",
+ "Fourth entry",
+ "Fifth entry",
+ "Sixth entry",
+ "Seventh entry",
+ "8",
+ "9",
+ "entry 10",
+ "entry 11",
+ "entry 12",
+ "entry-last",
+};
+
+int main()
+{
+ test_start("ctree");
+
+ ct_t * ctree = ct_new();
+
+ /* test adding values */
+ {
+ unsigned int i;
+ _assert (ctree->len == 0);
+ for (i = 0; i < num_entries; i++)
+ {
+ _assert (ct_add(ctree, entries[i], entries[i]) == 0);
+ }
+ }
+
+ /* test is the length is correct */
+ {
+ _assert (ctree->len == num_entries);
+ }
+
+ /* test adding duplicated values */
+ {
+ unsigned int i;
+ for (i = 0; i < num_entries; i++)
+ {
+ _assert (ct_add(ctree, entries[i], entries[i]) == CT_EXISTS);
+ }
+ }
+
+ /* test adding empty value */
+ {
+ _assert (ct_add(ctree, "", entries[0]) == CT_EXISTS);
+ }
+
+ /* test get */
+ {
+ unsigned int i;
+ for (i = 0; i < num_entries; i++)
+ {
+ _assert (ct_get(ctree, entries[i]) == entries[i]);
+ }
+ }
+
+ /* test getn */
+ {
+ unsigned int i;
+ for (i = 0; i < num_entries; i++)
+ {
+ _assert (ct_getn(
+ ctree,
+ entries[i],
+ strlen(entries[i])) == entries[i]);
+ }
+ }
+
+ /* test pop value */
+ {
+ unsigned int i;
+ for (i = 0; i < num_entries; i++)
+ {
+ _assert (ct_pop(ctree, entries[i]) == entries[i]);
+ }
+ }
+
+ /* test is the length is correct */
+ {
+ _assert (ctree->len == 0);
+ }
+
+ ct_free(ctree, NULL);
+
+ return test_end();
+}
--- /dev/null
+../src/expr/expr.c
--- /dev/null
+#include "../test.h"
+#include <expr/expr.h>
+
+
+int main()
+{
+ test_start("expr");
+
+ int64_t result;
+
+ _assert(expr_parse(&result, "5+37") == 0 && result == 42);
+ _assert(expr_parse(&result, "2+2*20") == 0 && result == 42);
+ _assert(expr_parse(&result, "(2+4)*7") == 0 && result == 42);
+ _assert(expr_parse(&result, "16%10*7") == 0 && result == 42);
+ _assert(expr_parse(&result, "7*14%56") == 0 && result == 42);
+ _assert(expr_parse(&result, "21/3*6") == 0 && result == 42);
+ _assert(expr_parse(&result, "22/3*6") == 0 && result == 42);
+
+ /* division by zero is not a good idea */
+ _assert(expr_parse(&result, "42/(2-2)") == EXPR_DIVISION_BY_ZERO);
+
+ /* module by zero is not a good idea either */
+ _assert(expr_parse(&result, "42%(2-2)") == EXPR_MODULO_BY_ZERO);
+
+ return test_end();
+}
--- /dev/null
+../src/siri/grammar/grammar.c
--- /dev/null
+#include "../test.h"
+#include <siri/grammar/grammar.h>
+#include <siri/grammar/gramp.h>
+
+#define assert_valid(__g, __q)(_assert(_is_valid(__g, __q)))
+#define assert_invalid(__g, __q)(_assert(!_is_valid(__g, __q)))
+
+static int _is_valid(cleri_grammar_t * grammar, char * query)
+{
+ cleri_parse_t * pr = cleri_parse(grammar, query);
+ _assert (pr);
+ int is_valid = pr->is_valid;
+ cleri_parse_free(pr);
+ return is_valid;
+}
+
+int main()
+{
+ test_start("grammar");
+
+ cleri_grammar_t * grammar = compile_siri_grammar_grammar();
+
+ assert_invalid(grammar, "select * from");
+ assert_invalid(grammar, "list");
+
+ assert_valid(grammar, "");
+ assert_valid(grammar, "now - 1w");
+ assert_valid(grammar, "help # with comments");
+ assert_valid(grammar, "select * from *");
+ assert_valid(grammar, "select * from 'series'");
+ assert_valid(grammar, "select * from * after now-1d");
+ assert_valid(grammar, "list series");
+ assert_valid(grammar,
+ "select mean(1h + 1m) from \"series-001\", \"series-002\", "
+ "\"series-003\" between 1360152000 and 1360152000 + 1d merge as "
+ "\"series\" using mean(1)");
+
+ cleri_grammar_free(grammar);
+
+ return test_end();
+}
--- /dev/null
+../src/imap/imap.c
+../src/vec/vec.c
+../src/logger/logger.c
\ No newline at end of file
--- /dev/null
+#include "../test.h"
+#include <inttypes.h>
+#include <imap/imap.h>
+
+static const unsigned int num_entries = 14;
+static char * entries[] = {
+ "Zero",
+ "First entry",
+ "Second entry",
+ "Third entry",
+ "Fourth entry",
+ "Fifth entry",
+ "Sixth entry",
+ "Seventh entry",
+ "8",
+ "9",
+ "entry 10",
+ "entry 11",
+ "entry 12",
+ "entry-last",
+};
+
+typedef struct
+{
+ uint32_t ref;
+ uint32_t id;
+} test_series_t;
+
+static test_series_t series_a = {
+ .ref=0,
+ .id=11
+};
+static test_series_t series_b = {
+ .ref=0,
+ .id=987
+};
+static test_series_t series_c = {
+ .ref=0,
+ .id=219
+};
+static test_series_t series_d = {
+ .ref=0,
+ .id=9
+};
+static test_series_t series_e = {
+ .ref=0,
+ .id=988
+};
+
+static imap_t * imap_dst;
+static imap_t * imap_tmp;
+
+static void test__imap_decref_cb(char * series)
+{
+ ((vec_object_t *) series)->ref--;
+}
+
+static int test__imap_id_count_cb(
+ char * series,
+ void * data __attribute__((unused)))
+{
+ return ((test_series_t *) series)->id;
+}
+
+static void test__imap_setup(void)
+{
+ imap_dst = imap_new();
+ imap_tmp = imap_new();
+
+ imap_set(imap_dst, series_a.id, &series_a);
+ series_a.ref++;
+
+ imap_set(imap_dst, series_b.id, &series_b);
+ series_b.ref++;
+
+ imap_set(imap_dst, series_c.id, &series_c);
+ series_c.ref++;
+
+ imap_set(imap_tmp, series_b.id, &series_b);
+ series_b.ref++;
+
+ imap_set(imap_tmp, series_c.id, &series_c);
+ series_c.ref++;
+
+ imap_set(imap_tmp, series_d.id, &series_d);
+ series_d.ref++;
+
+ imap_set(imap_tmp, series_e.id, &series_e);
+ series_e.ref++;
+}
+
+static int test_imap_add_set_get_pod(void)
+{
+ test_start("imap (add, set, get, pop)");
+
+ imap_t * imap = imap_new();
+
+ /* test is the length is correct */
+ _assert (imap->len == 0);
+
+ /* test imap_add */
+ {
+ unsigned int i;
+ for (i=0; i < num_entries; i++)
+ {
+ _assert (imap_add(imap, i, entries[i]) == 0);
+ }
+ _assert (imap->len == num_entries);
+ }
+
+ /* test imap_add duplicates */
+ {
+ unsigned int i;
+ for (i=0; i < num_entries; i++)
+ {
+ _assert (imap_add(imap, i, "duplicate") != 0);
+ }
+ }
+
+ /* test imap_get */
+ {
+ unsigned int i;
+ for (i=0; i < num_entries; i++)
+ {
+ _assert (imap_get(imap, i) == entries[i]);
+ }
+ }
+
+ /* test imap_set */
+ {
+ unsigned int i;
+ for (i=0; i < num_entries; i++)
+ {
+ unsigned int j = i * 2;
+ _assert (imap_set(
+ imap, j, entries[i]) == (int) (j < num_entries ? 0 : 1));
+ }
+
+ for (i=0; i < num_entries; i++)
+ {
+ unsigned int j = i * 2;
+ _assert (imap_get(imap, j) == entries[i]);
+ }
+ }
+
+ /* test imap_pop */
+ {
+ unsigned int i;
+ for (i=0; i < num_entries; i++)
+ {
+ unsigned int j = i * 2;
+ _assert (imap_pop(imap, j) == entries[i]);
+ }
+
+ for (i=1; i < num_entries; i+=2)
+ {
+ _assert (imap_pop(imap, i) == entries[i]);
+ }
+
+ _assert (imap->len == 0);
+ }
+
+ imap_free(imap, NULL);
+
+ return test_end();
+}
+
+static int test_imap_union()
+{
+ test_start("imap (union)");
+
+ test__imap_setup();
+
+ imap_union_ref(
+ imap_dst,
+ imap_tmp,
+ (imap_free_cb) test__imap_decref_cb);
+
+ _assert (imap_dst->len == 5);
+ _assert (imap_walk(
+ imap_dst,
+ (imap_cb) &test__imap_id_count_cb,
+ NULL) == (int) (
+ series_a.id +
+ series_b.id +
+ series_c.id +
+ series_d.id +
+ series_e.id));
+
+ imap_free(imap_dst, (imap_free_cb) test__imap_decref_cb);
+
+ _assert (series_a.ref == 0);
+ _assert (series_b.ref == 0);
+ _assert (series_c.ref == 0);
+ _assert (series_d.ref == 0);
+ _assert (series_e.ref == 0);
+
+ return test_end();
+}
+
+static int test_imap_intersection(void)
+{
+ test_start("imap (intersection)");
+
+ test__imap_setup();
+
+ imap_intersection_ref(
+ imap_dst,
+ imap_tmp,
+ (imap_free_cb) test__imap_decref_cb);
+
+ _assert (imap_dst->len == 2);
+ _assert (imap_walk(
+ imap_dst,
+ (imap_cb) &test__imap_id_count_cb,
+ NULL) == (int) (
+ series_b.id +
+ series_c.id));
+
+ imap_free(imap_dst, (imap_free_cb) test__imap_decref_cb);
+ _assert (series_a.ref == 0);
+ _assert (series_b.ref == 0);
+ _assert (series_c.ref == 0);
+ _assert (series_d.ref == 0);
+ _assert (series_e.ref == 0);
+
+ return test_end();
+}
+
+static int test_imap_difference(void)
+{
+ test_start("imap (difference)");
+
+ test__imap_setup();
+
+ imap_difference_ref(
+ imap_dst,
+ imap_tmp,
+ (imap_free_cb) test__imap_decref_cb);
+
+ _assert (imap_dst->len == 1);
+ _assert (imap_walk(
+ imap_dst,
+ (imap_cb) &test__imap_id_count_cb,
+ NULL) == (int) series_a.id);
+
+ imap_free(imap_dst, (imap_free_cb) test__imap_decref_cb);
+ _assert (series_a.ref == 0);
+ _assert (series_b.ref == 0);
+ _assert (series_c.ref == 0);
+ _assert (series_d.ref == 0);
+ _assert (series_e.ref == 0);
+
+ return test_end();
+}
+
+static int test_imap_symmetric_difference(void)
+{
+ test_start("imap (symmetric_difference)");
+
+ test__imap_setup();
+
+ imap_symmetric_difference_ref(
+ imap_dst,
+ imap_tmp,
+ (imap_free_cb) test__imap_decref_cb);
+
+ _assert (imap_dst->len == 3);
+ _assert (imap_walk(
+ imap_dst,
+ (imap_cb) &test__imap_id_count_cb,
+ NULL) == (int) (
+ series_a.id +
+ series_d.id +
+ series_e.id));
+
+ imap_free(imap_dst, (imap_free_cb) test__imap_decref_cb);
+ _assert (series_a.ref == 0);
+ _assert (series_b.ref == 0);
+ _assert (series_c.ref == 0);
+ _assert (series_d.ref == 0);
+ _assert (series_e.ref == 0);
+
+ return test_end();
+}
+
+int main()
+{
+ return (
+ test_imap_add_set_get_pod() ||
+ test_imap_union() ||
+ test_imap_intersection() ||
+ test_imap_difference() ||
+ test_imap_symmetric_difference() ||
+ 0
+ );
+}
--- /dev/null
+../src/iso8601/iso8601.c
--- /dev/null
+#include <time.h>
+#include <locale.h>
+#include "../test.h"
+#include <iso8601/iso8601.h>
+
+
+int main()
+{
+ test_start("iso8601");
+
+ /* Update local and timezone */
+ (void) setlocale(LC_ALL, "");
+ putenv("TZ=:UTC");
+ tzset();
+
+ iso8601_tz_t amsterdam = iso8601_tz("Europe/Amsterdam");
+ _assert(amsterdam > 0);
+
+ iso8601_tz_t utc = iso8601_tz("UTC");
+ _assert(utc > 0);
+
+ /* Some extra time zone test for case sensitivity etc. */
+ _assert(iso8601_tz("NAIVE") == 0);
+ _assert(iso8601_tz("europe/KIEV") > 0);
+ _assert(iso8601_tz("Ams") < 0);
+
+ /* Test parsing a year with time-zone information */
+ _assert(iso8601_parse_date("2013", amsterdam) == 1356994800);
+
+ /* Customer offset should be preferred over UTC */
+ _assert(iso8601_parse_date("2013+01", utc) == 1356994800);
+
+ /* UTC should be preferred over given time-zone (Amsterdam) */
+ _assert(iso8601_parse_date("2013Z", amsterdam) == 1356998400);
+
+ /* Test with minute precision */
+ _assert(iso8601_parse_date("2013-02-06T13:01:12", utc) == 1360155672);
+
+ /* Test short written */
+ _assert(iso8601_parse_date("2013-2-6 13:1:12", utc) == 1360155672);
+
+ /* Test summer time */
+ _assert(iso8601_parse_date("2016-04-21", amsterdam) == 1461189600);
+
+ /* Test error */
+ _assert(iso8601_parse_date("2016 04 21", amsterdam) == -1);
+
+ /* Test last day in year */
+ _assert(iso8601_parse_date("2013-12-31", utc) == 1388448000);
+
+ return test_end();
+}
--- /dev/null
+../src/siri/db/lookup.c
+../src/siri/err.c
+../src/logger/logger.c
--- /dev/null
+
+#include "../test.h"
+#include <siri/db/lookup.h>
+
+
+/* to at least 42 pools we devide series within 20% off from ideal */
+#define NPOOLS 42
+
+static double percentage_unequal = 0.2;
+static unsigned int countersa[NPOOLS];
+static unsigned int countersb[NPOOLS];
+
+static void init_counters(void)
+{
+ unsigned int i;
+ for (i = 0; i < NPOOLS; ++i)
+ {
+ countersa[i] = 0;
+ countersb[i] = 0;
+ }
+}
+
+int main()
+{
+ test_start("lookup");
+
+ unsigned int i;
+ siridb_lookup_t * lookup;
+ uint16_t match[30] = {
+ 0, 1, 0, 2, 3, 1, 0, 3, 3, 2, 2, 1, 0, 1, 0,
+ 2, 3, 1, 0, 3, 3, 2, 2, 1, 0, 1, 0, 2, 3, 1
+ };
+
+ /* make sure the lookup zie has not changed */
+ _assert(SIRIDB_LOOKUP_SZ == 8192);
+
+ {
+ unsigned int num_pools;
+ for (num_pools = 1; num_pools < NPOOLS; ++num_pools)
+ {
+ unsigned int n, p, ideal, lower, upper;
+ ideal = SIRIDB_LOOKUP_SZ / (num_pools * 2);
+ lower = ideal - (percentage_unequal * ideal);
+ upper = ideal + (percentage_unequal * ideal);
+ init_counters();
+ lookup = siridb_lookup_new(num_pools);
+ _assert (lookup);
+ for (n = 0; n < SIRIDB_LOOKUP_SZ; ++n)
+ {
+ /* check for SIRIDB_SERIES_IS_SERVER_ONE flag */
+ if ((_Bool) ((n / 11) % 2))
+ {
+ countersa[(*lookup)[n]]++;
+ }
+ else
+ {
+ countersb[(*lookup)[n]]++;
+ }
+ }
+ for (p = 0; p < num_pools; ++p)
+ {
+ _assert (countersa[p] >= lower && countersa[p] <= upper);
+ _assert (countersb[p] >= lower && countersb[p] <= upper);
+ }
+ free(lookup);
+ }
+ }
+
+ lookup = siridb_lookup_new(4);
+ _assert (lookup);
+
+ for (i = 0; i < 30; i++)
+ {
+ _assert( match[i] == (*lookup)[i] );
+ }
+
+ free(lookup);
+ return test_end();
+}
--- /dev/null
+../src/vec/vec.c
+../src/base64/base64.c
+../src/ctree/ctree.c
+../src/xpath/xpath.c
+../src/xmath/xmath.c
+../src/qpack/qpack.c
+../src/qpjson/qpjson.c
+../src/imap/imap.c
+../src/omap/omap.c
+../src/llist/llist.c
+../src/logger/logger.c
+../src/xstr/xstr.c
+../src/cfgparser/cfgparser.c
+../src/owcrypt/owcrypt.c
+../src/cexpr/cexpr.c
+../src/expr/expr.c
+../src/timeit/timeit.c
+../src/iso8601/iso8601.c
+../src/lib/http_parser.c
+../src/lock/lock.c
+../src/procinfo/procinfo.c
+../src/siri/api.c
+../src/siri/async.c
+../src/siri/backup.c
+../src/siri/buffersync.c
+../src/siri/err.c
+../src/siri/heartbeat.c
+../src/siri/optimize.c
+../src/siri/siri.c
+../src/siri/health.c
+../src/siri/version.c
+../src/siri/net/bserver.c
+../src/siri/net/clserver.c
+../src/siri/net/pkg.c
+../src/siri/net/promise.c
+../src/siri/net/promises.c
+../src/siri/net/protocol.c
+../src/siri/net/stream.c
+../src/siri/net/tcp.c
+../src/siri/net/pipe.c
+../src/siri/db/access.c
+../src/siri/db/aggregate.c
+../src/siri/db/auth.c
+../src/siri/db/buffer.c
+../src/siri/db/db.c
+../src/siri/db/ffile.c
+../src/siri/db/fifo.c
+../src/siri/db/forward.c
+../src/siri/db/group.c
+../src/siri/db/groups.c
+../src/siri/db/initsync.c
+../src/siri/db/insert.c
+../src/siri/db/listener.c
+../src/siri/db/lookup.c
+../src/siri/db/median.c
+../src/siri/db/misc.c
+../src/siri/db/nodes.c
+../src/siri/db/pcache.c
+../src/siri/db/points.c
+../src/siri/db/pool.c
+../src/siri/db/pools.c
+../src/siri/db/presuf.c
+../src/siri/db/props.c
+../src/siri/db/queries.c
+../src/siri/db/query.c
+../src/siri/db/re.c
+../src/siri/db/reindex.c
+../src/siri/db/replicate.c
+../src/siri/db/series.c
+../src/siri/db/server.c
+../src/siri/db/servers.c
+../src/siri/db/shard.c
+../src/siri/db/shards.c
+../src/siri/db/sset.c
+../src/siri/db/tag.c
+../src/siri/db/tags.c
+../src/siri/db/tasks.c
+../src/siri/db/tee.c
+../src/siri/db/time.c
+../src/siri/db/user.c
+../src/siri/db/users.c
+../src/siri/db/variance.c
+../src/siri/db/walker.c
+../src/siri/file/handler.c
+../src/siri/file/pointer.c
+../src/siri/service/account.c
+../src/siri/service/client.c
+../src/siri/service/request.c
+../src/siri/help/help.c
+../src/siri/cfg/cfg.c
+../src/siri/grammar/grammar.c
--- /dev/null
+#include "../test.h"
+#include <locale.h>
+#include <siri/db/series.h>
+#include <siri/db/shard.h>
+
+
+static int test_series_ensure_type(void)
+{
+ test_start("siridb (series_ensure_type)");
+ (void) setlocale(LC_NUMERIC, "English_Australia.1252");
+
+ siridb_series_t series;
+ qp_obj_t qp_obj;
+
+ /* test with integer series */
+ {
+ series.tp = TP_INT;
+
+ qp_obj.tp = QP_INT64;
+ qp_obj.via.int64 = -1;
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_INT64);
+ _assert (qp_obj.via.int64 == -1);
+
+ qp_obj.tp = QP_DOUBLE;
+ qp_obj.via.real = -1.0;
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_INT64);
+ _assert (qp_obj.via.int64 == -1);
+
+ qp_obj.tp = QP_RAW;
+ qp_obj.via.str = "55 percent";
+ qp_obj.len = strlen(qp_obj.via.str);
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_INT64);
+ _assert (qp_obj.via.int64 == 55);
+
+ qp_obj.tp = QP_RAW;
+ qp_obj.via.str = "garbage";
+ qp_obj.len = strlen(qp_obj.via.str);
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_INT64);
+ _assert (qp_obj.via.int64 == 0);
+ }
+
+ /* test with double series */
+ {
+ series.tp = TP_DOUBLE;
+
+ qp_obj.tp = QP_DOUBLE;
+ qp_obj.via.real = -1.1;
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_DOUBLE);
+ _assert (qp_obj.via.real == -1.1);
+
+ qp_obj.tp = QP_INT64;
+ qp_obj.via.int64 = -1;
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_DOUBLE);
+ _assert (qp_obj.via.real == -1.0);
+
+ qp_obj.tp = QP_RAW;
+ qp_obj.via.str = "0.5 percent";
+ qp_obj.len = strlen(qp_obj.via.str);
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_DOUBLE);
+ _assert (qp_obj.via.real == 0.5);
+
+ qp_obj.tp = QP_RAW;
+ qp_obj.via.str = "garbage";
+ qp_obj.len = strlen(qp_obj.via.str);
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_DOUBLE);
+ _assert (qp_obj.via.real == 0.0);
+ }
+
+ /* test with string series */
+ {
+ series.tp = TP_STRING;
+
+ qp_obj.tp = QP_RAW;
+ qp_obj.via.str = "55.3 percent";
+ qp_obj.len = strlen(qp_obj.via.str);
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_RAW);
+ _assert (strlen("55.3 percent") == qp_obj.len);
+ _assert (strncmp("55.3 percent", qp_obj.via.str, qp_obj.len) == 0);
+
+ qp_obj.tp = QP_RAW;
+ qp_obj.via.str = "";
+ qp_obj.len = strlen(qp_obj.via.str);
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_RAW);
+ _assert (strlen("") == qp_obj.len);
+ _assert (strncmp("", qp_obj.via.str, qp_obj.len) == 0);
+
+ qp_obj.tp = QP_DOUBLE;
+ qp_obj.via.real = -1.1;
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_RAW);
+ _assert (strlen("-1.100000") == qp_obj.len);
+ _assert (strncmp("-1.100000", qp_obj.via.str, qp_obj.len) == 0);
+
+ qp_obj.tp = QP_INT64;
+ qp_obj.via.int64 = -1;
+ siridb_series_ensure_type(&series, &qp_obj);
+ _assert (qp_obj.tp == QP_RAW);
+ _assert (strlen("-1") == qp_obj.len);
+ _assert (strncmp("-1", qp_obj.via.str, qp_obj.len) == 0);
+ }
+
+ (void) setlocale(LC_ALL, NULL);
+ return test_end();
+};
+
+int main()
+{
+ return (
+ test_series_ensure_type() ||
+ 0
+ );
+};
\ No newline at end of file
--- /dev/null
+../src/vec/vec.c
+../src/logger/logger.c
\ No newline at end of file
--- /dev/null
+#include "../test.h"
+#include <vec/vec.h>
+
+const unsigned int num_entries = 14;
+char * entries[] = {
+ "Zero",
+ "First entry",
+ "Second entry",
+ "Third entry",
+ "Fourth entry",
+ "Fifth entry",
+ "Sixth entry",
+ "Seventh entry",
+ "8",
+ "9",
+ "entry 10",
+ "entry 11",
+ "entry 12",
+ "entry-last"
+};
+
+int main()
+{
+ test_start("vec");
+
+ /* vec_append_safe */
+ {
+ vec_t * vec = vec_new(0);
+ _assert (vec->len == 0);
+ _assert (vec->size == 0);
+
+ unsigned int i;
+ for (i = 0; i < num_entries; i++)
+ {
+ _assert (vec_append_safe(&vec, entries[i]) == 0);
+ }
+
+ /* vec_copy */
+ {
+ vec_t * veccp = vec_copy(vec);
+ unsigned int i;
+ for (i = 0; i < num_entries; i++)
+ {
+ _assert (veccp->data[i] == entries[i]);
+ }
+ vec_free(veccp);
+ }
+
+ _assert (vec->len == num_entries);
+ vec_free(vec);
+ }
+
+ /* vec_append */
+ {
+ vec_t * vec = vec_new(num_entries);
+ _assert (vec->len == 0);
+ _assert (vec->size == num_entries);
+
+ unsigned int i;
+ for (i = 0; i < num_entries; i++)
+ {
+ vec_append(vec, entries[i]);
+ }
+
+ _assert (vec->len == num_entries);
+
+ /* vec_pop */
+ for (i = num_entries; i-- > 0;)
+ {
+ _assert (vec_pop(vec) == entries[i]);
+ }
+
+ vec_free(vec);
+ }
+
+
+ return test_end();
+}
--- /dev/null
+../src/siri/version.c
--- /dev/null
+#include <stdio.h>
+#include <stdlib.h>
+#include "../test.h"
+#include <siri/version.h>
+
+
+
+int old_version_cmp(const char * version_a, const char * version_b)
+{
+ long int a, b;
+
+ char * str_a = (char *) version_a;
+ char * str_b = (char *) version_b;
+
+ while (1)
+ {
+ a = strtol(str_a, &str_a, 10);
+ b = strtol(str_b, &str_b, 10);
+
+ if (a != b)
+ {
+ return a - b;
+ }
+ else if (!*str_a && !*str_b)
+ {
+ return 0;
+ }
+ else if (!*str_a)
+ {
+ return -1;
+ }
+ else if (!*str_b)
+ {
+ return 1;
+ }
+ str_a++;
+ str_b++;
+ }
+
+ /* we should NEVER get here */
+ _assert (0);
+ return 0;
+}
+
+
+int main()
+{
+ test_start("version");
+
+ /* alpha verioning should be ignored by version compare */
+ _assert (siri_version_cmp("1.0.0", "2.0.0") < 0);
+ _assert (siri_version_cmp("1.0.0", "2.0.0-alpha-0") < 0);
+ _assert (siri_version_cmp("2.0.0", "1.0.0") > 0);
+ _assert (siri_version_cmp("2.0.0", "1.0.0") > 0);
+ _assert (siri_version_cmp("2.2.0", "2.32.0") < 0);
+ _assert (siri_version_cmp("2.32.0", "2.2.0") > 0);
+ _assert (siri_version_cmp("2.0.5", "2.0.22") < 0);
+ _assert (siri_version_cmp("2.0.22", "2.0.5") > 0);
+ _assert (siri_version_cmp("2.0.5-alpha-0", "2.0.22") < 0);
+ _assert (siri_version_cmp("2.0.22-alpha-0", "2.0.5") > 0);
+ _assert (siri_version_cmp("a", "") == 0);
+ _assert (siri_version_cmp("", "b") == 0);
+ _assert (siri_version_cmp("", "") == 0);
+ _assert (siri_version_cmp("2.0.30", "2.0.30") == 0);
+ _assert (siri_version_cmp("2.0.30-alpha-1-debug", "2.0.30") == 0);
+ _assert (siri_version_cmp("2.0.30-alpha-1", "2.0.30-alpha-0") == 0);
+
+ /* old version compare function should not break with -alpha versions */
+ _assert (old_version_cmp("2.0.5-alpha-0", "2.0.22") < 0);
+ _assert (old_version_cmp("2.0.22-alpha-0", "2.0.5") > 0);
+ _assert (old_version_cmp("2.0.30-alpha-1-debug", "2.0.30") > 0);
+ /* This last one is < 0 since -1 < -0 */
+ _assert (old_version_cmp("2.0.30-alpha-1", "2.0.30-alpha-0") < 0);
+
+ return test_end();
+}
--- /dev/null
+../src/xmath/xmath.c
--- /dev/null
+#include "../test.h"
+#include <xmath/xmath.h>
+
+
+int main()
+{
+ test_start("xmath");
+
+ /* xmath_ipow */
+ {
+ _assert (xmath_ipow(1000, 0) == 1);
+ _assert (xmath_ipow(1000, 1) == 1000);
+ _assert (xmath_ipow(1000, 2) == 1000000);
+ _assert (xmath_ipow(1000, 3) == 1000000000);
+ _assert (xmath_ipow(2, 8) == 256);
+ }
+
+ /* xmath_max_size */
+ {
+ _assert (xmath_max_size(3, 10, 20, 30) == 30);
+ _assert (xmath_max_size(3, 30, 20, 10) == 30);
+ _assert (xmath_max_size(3, 10, 30, 20) == 30);
+ }
+
+ return test_end();
+}
--- /dev/null
+../src/xpath/xpath.c
+../src/logger/logger.c
\ No newline at end of file
--- /dev/null
+#include "../test.h"
+#include <xpath/xpath.h>
+
+
+int main()
+{
+ test_start("xpath");
+
+ /* xpath_file_exist */
+ {
+ _assert (xpath_file_exist("./test.h"));
+ _assert (!xpath_file_exist("./test.foo"));
+ }
+
+ return test_end();
+}
--- /dev/null
+../src/xstr/xstr.c
--- /dev/null
+#include "../test.h"
+#include <xstr/xstr.h>
+
+
+int main()
+{
+ test_start("xstr");
+
+ /* xstr_to_double */
+ {
+ _assert (xstr_to_double("0.5") == 0.5);
+ _assert (xstr_to_double("0.55") == 0.55);
+ _assert (xstr_to_double("123.456") == 123.456);
+ _assert (xstr_to_double("123") == 123.0);
+ _assert (xstr_to_double("123.") == 123.0);
+ _assert (xstr_to_double("123.") == 123.0);
+ _assert (xstr_to_double("+1234") == 1234.0);
+ _assert (xstr_to_double("-1234") == -1234.0);
+ _assert (xstr_to_double("123456.") == 123456.0);
+ _assert (xstr_to_double("-0.5") == -0.5);
+ _assert (xstr_to_double("-0.56") == -0.56);
+ _assert (xstr_to_double("+0.5") == 0.5);
+ _assert (xstr_to_double("-.5") == -0.5);
+ _assert (xstr_to_double("+.55") == 0.55);
+ _assert (xstr_to_double(".55") == 0.55);
+ _assert (xstr_to_double("-.55") == -0.55);
+ }
+
+ return test_end();
+}
projects / siridb-server.git / commitdiff
