Add ignore_broken_data config option

The option will cause SiriDB to ignore broken or corrupted data that would otherwise prevent the server or database from fully loading.

This is useful for transient data on systems prone to hard resets, which might leave the database in an inconsistent state.

diff --git a/include/siri/cfg/cfg.h b/include/siri/cfg/cfg.h
index a5cfe90f4dfa9767430f11b5cdba8bb0339276e9..8670ece1d911555fe0c1f60563ff43c4f5c83b87 100644 (file)
--- a/include/siri/cfg/cfg.h
+++ b/include/siri/cfg/cfg.h
@@ -45,6 +45,8 @@ struct siri_cfg_s
     char server_address[SIRI_CFG_MAX_LEN_ADDRESS];
     char db_path[XPATH_MAX];
     char pipe_client_name[XPATH_MAX];
+
+    uint8_t ignore_broken_data;
 };
 
 #endif  /* SIRI_CFG_H_ */

diff --git a/siridb.conf b/siridb.conf
index 71ed0726c993f1ac17117dd44dea5e306c4c03bd..d8d271d691411933b11d5eabd0f201eb937f20ed 100644 (file)
--- a/siridb.conf
+++ b/siridb.conf
@@ -63,7 +63,7 @@ buffer_sync_interval = 0
 
 #
 # SiriDB will not open more shard files than max_open_files. Note that the
-# total number of open files can be sligtly higher since SiriDB also needs
+# total number of open files can be slightly higher since SiriDB also needs
 # a few other files to write to.
 #
 max_open_files = 32768
@@ -81,6 +81,15 @@ enable_shard_compression = 1
 #
 enable_shard_auto_duration = 1
 
+#
+# SiriDB will ignore corrupted or broken shards and related database files even
+# at the cost of losing some or all data.
+#
+# This option is useful especially for transient data on systems prone to power
+# outage or frequent hard resets.
+#
+ignore_broken_data = 0
+
 #
 # Enable named pipe support for client connections.
 #

diff --git a/src/siri/cfg/cfg.c b/src/siri/cfg/cfg.c
index 5e3b7961717ff0f2e6a1a26fa93c9a548b96aaeb..7b7736397f8cafe3764c23615332f4f9d9408ad5 100644 (file)
--- a/src/siri/cfg/cfg.c
+++ b/src/siri/cfg/cfg.c
@@ -32,6 +32,7 @@ static siri_cfg_t siri_cfg = {
         .pipe_support=0,
         .pipe_client_name="siridb_client.sock",
         .buffer_sync_interval=0,
+        .ignore_broken_data=0
 };
 
 static void SIRI_CFG_read_uint(
@@ -56,6 +57,7 @@ static void SIRI_CFG_read_ip_support(cfgparser_t * cfgparser);
 static void SIRI_CFG_read_shard_compression(cfgparser_t * cfgparser);
 static void SIRI_CFG_read_shard_auto_duration(cfgparser_t * cfgparser);
 static void SIRI_CFG_read_pipe_support(cfgparser_t * cfgparser);
+static void SIRI_CFG_ignore_broken_data(cfgparser_t * cfgparser);
 
 void siri_cfg_init(siri_t * siri)
 {
@@ -160,6 +162,8 @@ void siri_cfg_init(siri_t * siri)
             &tmp);
     siri_cfg.buffer_sync_interval = (uint32_t) tmp;
 
+    SIRI_CFG_ignore_broken_data(cfgparser);
+
     cfgparser_free(cfgparser);
 }
 
@@ -354,6 +358,32 @@ static void SIRI_CFG_read_pipe_support(cfgparser_t * cfgparser)
     }
 }
 
+static void SIRI_CFG_ignore_broken_data(cfgparser_t * cfgparser)
+{
+    cfgparser_option_t * option;
+    cfgparser_return_t rc;
+    rc = cfgparser_get_option(
+        &option,
+        cfgparser,
+        "siridb",
+        "ignore_broken_data");
+    if (rc != CFGPARSER_SUCCESS)
+    {
+        return; // optional config option
+    }
+    else if (option->tp != CFGPARSER_TP_INTEGER || option->val->integer > 1)
+    {
+        log_warning(
+            "Error reading 'ignore_broken_data' in '%s': %s.",
+            siri.args->config,
+            "error: expecting 0 or 1");
+    }
+    else if (option->val->integer == 1)
+    {
+        siri_cfg.ignore_broken_data = 1;
+    }
+}
+
 static void SIRI_CFG_read_addr(
         cfgparser_t * cfgparser,
         const char * option_name,

diff --git a/src/siri/db/buffer.c b/src/siri/db/buffer.c
index 67d405ed31c28083dc1520819ad030daf2d11abf..e80e51d3ae563274afa930e3ec909eec6ba83f0e 100644 (file)
--- a/src/siri/db/buffer.c
+++ b/src/siri/db/buffer.c
@@ -16,6 +16,7 @@
 #include <unistd.h>
 #include <xpath/xpath.h>
 #include <assert.h>
+#include <stdbool.h>
 
 #define SIRIDB_BUFFER_FN "buffer.dat"
 
@@ -255,9 +256,10 @@ int siridb_buffer_load(siridb_t * siridb)
     char * buf, * pt;
     long int offset = 0;
     siridb_series_t * series;
-    _Bool log_migrate = 1;
+    bool log_migrate = true;
     uint32_t buf_start, series_id;
     uint64_t * ts;
+    uint8_t ignore_broken_data = siri.cfg->ignore_broken_data;
 
     log_info("Loading and cleanup buffer");
 
@@ -287,11 +289,23 @@ int siridb_buffer_load(siridb_t * siridb)
 
     if (xpath_file_exist(fn_temp))
     {
-        free(buf);
-        log_error(
+        if (ignore_broken_data)
+        {
+            log_error("Temporary buffer file found: '%s'. Removing...", fn_temp);
+            if (unlink(fn_temp))
+            {
+                free(buf);
+                log_error("Failed to remove temporary buffer: %s", fn_temp);
+                return -1;
+            }
+        } else
+        {
+            free(buf);
+            log_error(
                 "Temporary buffer file found: '%s'. "
                 "Check if something went wrong or remove this file", fn_temp);
-        return -1;
+            return -1;
+        }
     }
 
     if ((fp = fopen(fn, "r")) == NULL)
@@ -326,7 +340,7 @@ int siridb_buffer_load(siridb_t * siridb)
                 if (log_migrate)
                 {
                     log_warning("Buffer will be migrated");
-                    log_migrate = 0;
+                    log_migrate = false;
                 }
                 buffer__migrate_to_new(pt, cur_size);
             }

diff --git a/src/siri/db/shards.c b/src/siri/db/shards.c
index 6eff024591a3d1cf18dbd5878a212c8ba5a743ab..752a25766c304fc635ef87b67d24a6512f36585f 100644 (file)
--- a/src/siri/db/shards.c
+++ b/src/siri/db/shards.c
@@ -111,6 +111,26 @@ static bool SHARDS_is_temp_fn(char * fn)
     )));
 }
 
+static bool SHARDS_remove_shard_file(const char * path, const char * fn)
+{
+    siridb_misc_get_fn(shard_path, path, fn);
+    if (unlink(shard_path))
+    {
+        log_error("Error while removing shard file: '%s'", shard_path);
+        return false;
+    }
+
+    siridb_shard_idx_file(index_path, shard_path);
+    if (xpath_file_exist(index_path) && unlink(index_path))
+    {
+        log_error("Error while removing index file: '%s'", index_path);
+        return false;
+    }
+
+    log_warning("Shard file '%s' removed", fn);
+    return true;
+}
+
 
 /*
  * Returns 0 if successful or -1 in case of an error.
@@ -123,6 +143,7 @@ int siridb_shards_load(siridb_t * siridb)
     char buffer[XPATH_MAX];
     int n, total, rc = 0;
     uint64_t shard_id, duration;
+    bool ignore_broken_data = siri.cfg->ignore_broken_data;
 
     memset(&st, 0, sizeof(struct stat));
 
@@ -190,8 +211,13 @@ int siridb_shards_load(siridb_t * siridb)
                 if (siridb_shard_migrate(siridb, shard_id, &duration))
                 {
                     log_error("Error while migrating shard: '%s'", base_fn);
-                    rc = -1;
-                    break;
+                    if (!ignore_broken_data || !SHARDS_remove_shard_file(path, base_fn))
+                    {
+                        rc = -1;
+                        break;
+                    } else {
+                        continue;
+                    }
                 }
             }
             else
@@ -204,8 +230,13 @@ int siridb_shards_load(siridb_t * siridb)
         if (siridb_shard_load(siridb, shard_id, duration))
         {
            log_error("Error while loading shard: '%s'", base_fn);
-           rc = -1;
-           break;
+            if (!ignore_broken_data || !SHARDS_remove_shard_file(path, base_fn))
+            {
+                rc = -1;
+                break;
+            } else {
+                continue;
+            }
         }
     }