From: Michal Skvely Date: Mon, 29 Mar 2021 12:59:50 +0000 (+0200) Subject: Add ignore_broken_data config option X-Git-Tag: archive/raspbian/2.0.44-1+rpi1^2~3^2^2~3^2 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=a48d6496eb4943f510af549dfb7f5bf24ded6e4e;p=siridb-server.git Add ignore_broken_data config option The option will cause SiriDB to ignore broken or corrupted data that would otherwise prevent the server or database from fully loading. This is useful for transient data on systems prone to hard resets, which might leave the database in an inconsistent state. --- diff --git a/include/siri/cfg/cfg.h b/include/siri/cfg/cfg.h index a5cfe90f..8670ece1 100644 --- a/include/siri/cfg/cfg.h +++ b/include/siri/cfg/cfg.h @@ -45,6 +45,8 @@ struct siri_cfg_s char server_address[SIRI_CFG_MAX_LEN_ADDRESS]; char db_path[XPATH_MAX]; char pipe_client_name[XPATH_MAX]; + + uint8_t ignore_broken_data; }; #endif /* SIRI_CFG_H_ */ diff --git a/siridb.conf b/siridb.conf index 71ed0726..d8d271d6 100644 --- a/siridb.conf +++ b/siridb.conf @@ -63,7 +63,7 @@ buffer_sync_interval = 0 # # SiriDB will not open more shard files than max_open_files. Note that the -# total number of open files can be sligtly higher since SiriDB also needs +# total number of open files can be slightly higher since SiriDB also needs # a few other files to write to. # max_open_files = 32768 @@ -81,6 +81,15 @@ enable_shard_compression = 1 # enable_shard_auto_duration = 1 +# +# SiriDB will ignore corrupted or broken shards and related database files even +# at the cost of losing some or all data. +# +# This option is useful especially for transient data on systems prone to power +# outage or frequent hard resets. +# +ignore_broken_data = 0 + # # Enable named pipe support for client connections. # diff --git a/src/siri/cfg/cfg.c b/src/siri/cfg/cfg.c index 5e3b7961..7b773639 100644 --- a/src/siri/cfg/cfg.c +++ b/src/siri/cfg/cfg.c @@ -32,6 +32,7 @@ static siri_cfg_t siri_cfg = { .pipe_support=0, .pipe_client_name="siridb_client.sock", .buffer_sync_interval=0, + .ignore_broken_data=0 }; static void SIRI_CFG_read_uint( @@ -56,6 +57,7 @@ static void SIRI_CFG_read_ip_support(cfgparser_t * cfgparser); static void SIRI_CFG_read_shard_compression(cfgparser_t * cfgparser); static void SIRI_CFG_read_shard_auto_duration(cfgparser_t * cfgparser); static void SIRI_CFG_read_pipe_support(cfgparser_t * cfgparser); +static void SIRI_CFG_ignore_broken_data(cfgparser_t * cfgparser); void siri_cfg_init(siri_t * siri) { @@ -160,6 +162,8 @@ void siri_cfg_init(siri_t * siri) &tmp); siri_cfg.buffer_sync_interval = (uint32_t) tmp; + SIRI_CFG_ignore_broken_data(cfgparser); + cfgparser_free(cfgparser); } @@ -354,6 +358,32 @@ static void SIRI_CFG_read_pipe_support(cfgparser_t * cfgparser) } } +static void SIRI_CFG_ignore_broken_data(cfgparser_t * cfgparser) +{ + cfgparser_option_t * option; + cfgparser_return_t rc; + rc = cfgparser_get_option( + &option, + cfgparser, + "siridb", + "ignore_broken_data"); + if (rc != CFGPARSER_SUCCESS) + { + return; // optional config option + } + else if (option->tp != CFGPARSER_TP_INTEGER || option->val->integer > 1) + { + log_warning( + "Error reading 'ignore_broken_data' in '%s': %s.", + siri.args->config, + "error: expecting 0 or 1"); + } + else if (option->val->integer == 1) + { + siri_cfg.ignore_broken_data = 1; + } +} + static void SIRI_CFG_read_addr( cfgparser_t * cfgparser, const char * option_name, diff --git a/src/siri/db/buffer.c b/src/siri/db/buffer.c index 67d405ed..e80e51d3 100644 --- a/src/siri/db/buffer.c +++ b/src/siri/db/buffer.c @@ -16,6 +16,7 @@ #include #include #include +#include #define SIRIDB_BUFFER_FN "buffer.dat" @@ -255,9 +256,10 @@ int siridb_buffer_load(siridb_t * siridb) char * buf, * pt; long int offset = 0; siridb_series_t * series; - _Bool log_migrate = 1; + bool log_migrate = true; uint32_t buf_start, series_id; uint64_t * ts; + uint8_t ignore_broken_data = siri.cfg->ignore_broken_data; log_info("Loading and cleanup buffer"); @@ -287,11 +289,23 @@ int siridb_buffer_load(siridb_t * siridb) if (xpath_file_exist(fn_temp)) { - free(buf); - log_error( + if (ignore_broken_data) + { + log_error("Temporary buffer file found: '%s'. Removing...", fn_temp); + if (unlink(fn_temp)) + { + free(buf); + log_error("Failed to remove temporary buffer: %s", fn_temp); + return -1; + } + } else + { + free(buf); + log_error( "Temporary buffer file found: '%s'. " "Check if something went wrong or remove this file", fn_temp); - return -1; + return -1; + } } if ((fp = fopen(fn, "r")) == NULL) @@ -326,7 +340,7 @@ int siridb_buffer_load(siridb_t * siridb) if (log_migrate) { log_warning("Buffer will be migrated"); - log_migrate = 0; + log_migrate = false; } buffer__migrate_to_new(pt, cur_size); } diff --git a/src/siri/db/shards.c b/src/siri/db/shards.c index 6eff0245..752a2576 100644 --- a/src/siri/db/shards.c +++ b/src/siri/db/shards.c @@ -111,6 +111,26 @@ static bool SHARDS_is_temp_fn(char * fn) ))); } +static bool SHARDS_remove_shard_file(const char * path, const char * fn) +{ + siridb_misc_get_fn(shard_path, path, fn); + if (unlink(shard_path)) + { + log_error("Error while removing shard file: '%s'", shard_path); + return false; + } + + siridb_shard_idx_file(index_path, shard_path); + if (xpath_file_exist(index_path) && unlink(index_path)) + { + log_error("Error while removing index file: '%s'", index_path); + return false; + } + + log_warning("Shard file '%s' removed", fn); + return true; +} + /* * Returns 0 if successful or -1 in case of an error. @@ -123,6 +143,7 @@ int siridb_shards_load(siridb_t * siridb) char buffer[XPATH_MAX]; int n, total, rc = 0; uint64_t shard_id, duration; + bool ignore_broken_data = siri.cfg->ignore_broken_data; memset(&st, 0, sizeof(struct stat)); @@ -190,8 +211,13 @@ int siridb_shards_load(siridb_t * siridb) if (siridb_shard_migrate(siridb, shard_id, &duration)) { log_error("Error while migrating shard: '%s'", base_fn); - rc = -1; - break; + if (!ignore_broken_data || !SHARDS_remove_shard_file(path, base_fn)) + { + rc = -1; + break; + } else { + continue; + } } } else @@ -204,8 +230,13 @@ int siridb_shards_load(siridb_t * siridb) if (siridb_shard_load(siridb, shard_id, duration)) { log_error("Error while loading shard: '%s'", base_fn); - rc = -1; - break; + if (!ignore_broken_data || !SHARDS_remove_shard_file(path, base_fn)) + { + rc = -1; + break; + } else { + continue; + } } }