From: Timo Sirainen Date: Thu, 30 Apr 2026 12:11:57 +0000 (+0000) Subject: [PATCH 07/12] lib-index: Add keyed xxh64 strmap format v2, gated by config version X-Git-Tag: archive/raspbian/1%2.4.1+dfsg1-6+rpi1+deb13u7^2~34 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=6a450a45f1ca4726d2720fb48db6f83989e002fd;p=dovecot.git [PATCH 07/12] lib-index: Add keyed xxh64 strmap format v2, gated by config version The on-disk hash stored in the strmap file is now optionally a keyed xxh64_to_32() with a per-file random 64-bit IV stored in the file header. This replaces the previous plain crc32_str_nonzero() output and hardens the strmap against deliberately-collided message-id hashes. The v1 (crc32) format remains fully readable and writable so older configurations keep working without forced rebuilds. On-disk: - v1 header is 8 bytes: version, 3 unused, uid_validity (unchanged). - v2 header is 16 bytes: version, compat_flags, 2 unused, uid_validity, hash_iv. The first 8 bytes of v2 align with v1 byte-for-byte, so the open path reads 8 bytes, dispatches on version, and reads the trailing 8 bytes only for v2. Runtime: - strmap->enable_xxh64 selects the format used when creating a new file (or recreating one whose uid_validity has changed). - view->format_version follows the on-disk file when one exists; for fresh files it follows the strmap-wide preference. - Renumber recreate_write() preserves view->format_version, so we never silently migrate an existing file across the threshold while it is still in use - the migration only happens at open time. - When enable_xxh64 is TRUE and the on-disk file is v1, open treats it as a version mismatch: the file is unlinked and the next sync's recreate_write() produces a v2 file. Below the threshold v1 stays v1 indefinitely. - strmap_hash_str() dispatches per-view, so v1 files keep using crc32 hashes and v2 files use keyed xxh64 even in mixed setups. Gbp-Pq: Name 0007-lib-index-Add-keyed-xxh64-strmap-format-v2-gated-by-.patch --- diff --git a/src/lib-index/mail-index-strmap.c b/src/lib-index/mail-index-strmap.c index 254080d..10c8351 100644 --- a/src/lib-index/mail-index-strmap.c +++ b/src/lib-index/mail-index-strmap.c @@ -8,6 +8,8 @@ #include "file-lock.h" #include "file-dotlock.h" #include "crc32.h" +#include "randgen.h" +#include "xxh64.h" #include "safe-mkstemp.h" #include "str.h" #include "mail-index-private.h" @@ -24,6 +26,12 @@ struct mail_index_strmap { struct file_lock *file_lock; struct dotlock *dotlock; struct dotlock_settings dotlock_settings; + + /* If TRUE, new files are created in v2 format (xxh64-keyed hashes). + If FALSE, new files are created in v1 format (crc32). Existing + files of either version are always read; on rebuild they are + recreated in this preferred format. */ + bool enable_xxh64; }; struct mail_index_strmap_view { @@ -34,6 +42,14 @@ struct mail_index_strmap_view { ARRAY(uint32_t) recs_hash; struct hash2_table *hash; + /* On-disk format version that view's hash values are computed + against. v1 = crc32_str_nonzero, v2 = keyed xxh64. Set from the + file header when opening an existing file; otherwise from + strmap->enable_xxh64. */ + uint8_t format_version; + /* Per-file random IV mixed into the xxh64 hash. Unused for v1. */ + uint64_t hash_iv; + mail_index_strmap_key_cmp_t *key_compare; mail_index_strmap_rec_cmp_t *rec_compare; mail_index_strmap_remap_t *remap_cb; @@ -92,7 +108,8 @@ static const struct dotlock_settings default_dotlock_settings = { }; struct mail_index_strmap * -mail_index_strmap_init(struct mail_index *index, const char *suffix) +mail_index_strmap_init(struct mail_index *index, const char *suffix, + bool enable_xxh64) { struct mail_index_strmap *strmap; @@ -102,6 +119,7 @@ mail_index_strmap_init(struct mail_index *index, const char *suffix) strmap->index = index; strmap->path = i_strconcat(index->filepath, suffix, NULL); strmap->fd = -1; + strmap->enable_xxh64 = enable_xxh64; strmap->dotlock_settings = default_dotlock_settings; strmap->dotlock_settings.use_excl_lock = @@ -198,6 +216,9 @@ mail_index_strmap_view_open(struct mail_index_strmap *strmap, i_array_init(&view->recs, 64); i_array_init(&view->recs_hash, 64); + view->format_version = strmap->enable_xxh64 ? + MAIL_INDEX_STRMAP_VERSION_V2 : MAIL_INDEX_STRMAP_VERSION_V1; + random_fill(&view->hash_iv, sizeof(view->hash_iv)); view->hash = hash2_create(0, sizeof(struct mail_index_strmap_rec), mail_index_strmap_hash_key, mail_index_strmap_hash_cmp, view); @@ -250,7 +271,7 @@ static int mail_index_strmap_open(struct mail_index_strmap_view *view) const struct mail_index_header *idx_hdr; struct mail_index_strmap_header hdr; const unsigned char *data; - size_t size; + size_t size, hdr_size; int ret; i_assert(strmap->fd == -1); @@ -263,7 +284,12 @@ static int mail_index_strmap_open(struct mail_index_strmap_view *view) return -1; } strmap->input = i_stream_create_fd(strmap->fd, SIZE_MAX); - ret = i_stream_read_bytes(strmap->input, &data, &size, sizeof(hdr)); + + /* Read the smaller v1 header first; bytes 0..7 have identical + layout in both formats so we can decide based on the version + byte before reading the v2 tail. */ + ret = i_stream_read_bytes(strmap->input, &data, &size, + MAIL_INDEX_STRMAP_HEADER_V1_SIZE); if (ret <= 0) { if (ret < 0) { mail_index_strmap_set_syscall_error(strmap, "read()"); @@ -274,23 +300,55 @@ static int mail_index_strmap_open(struct mail_index_strmap_view *view) } return ret; } - memcpy(&hdr, data, sizeof(hdr)); + + i_zero(&hdr); + hdr_size = data[0] == MAIL_INDEX_STRMAP_VERSION_V2 ? + MAIL_INDEX_STRMAP_HEADER_V2_SIZE : + MAIL_INDEX_STRMAP_HEADER_V1_SIZE; + if (hdr_size > MAIL_INDEX_STRMAP_HEADER_V1_SIZE) { + ret = i_stream_read_bytes(strmap->input, &data, &size, hdr_size); + if (ret <= 0) { + if (ret < 0) { + mail_index_strmap_set_syscall_error(strmap, "read()"); + mail_index_strmap_close(strmap); + } else { + mail_index_strmap_view_set_corrupted(view); + } + return ret; + } + } + memcpy(&hdr, data, hdr_size); idx_hdr = mail_index_get_header(view->view); - if (hdr.version != MAIL_INDEX_STRMAP_VERSION || - hdr.uid_validity != idx_hdr->uid_validity) { + uint8_t compat_flags = 0; +#ifndef WORDS_BIGENDIAN + if (hdr.version >= MAIL_INDEX_STRMAP_VERSION_V2) + compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN; +#endif + if ((hdr.version != MAIL_INDEX_STRMAP_VERSION_V1 && + hdr.version != MAIL_INDEX_STRMAP_VERSION_V2) || + hdr.compat_flags != compat_flags || + hdr.uid_validity != idx_hdr->uid_validity || + (strmap->enable_xxh64 && + hdr.version < MAIL_INDEX_STRMAP_VERSION_V2)) { /* need to rebuild. if we already had something in the strmap, - we can keep it. */ + we can keep it. The enable_xxh64 case forces a v1 file to + be discarded so the next write creates a v2 file - we never + stay on v1 once the dovecot_storage_version threshold has + been reached. */ i_unlink(strmap->path); mail_index_strmap_close(strmap); return 0; } + view->format_version = hdr.version; + view->hash_iv = hdr.version >= MAIL_INDEX_STRMAP_VERSION_V2 ? + hdr.hash_iv : 0; /* we'll read the entire file from the beginning */ view->last_added_uid = 0; view->last_read_uid = 0; view->total_ref_count = 0; - view->last_read_block_offset = sizeof(struct mail_index_strmap_header); + view->last_read_block_offset = hdr_size; view->next_str_idx = 1; mail_index_strmap_view_reset(view); @@ -750,6 +808,17 @@ static inline uint32_t crc32_str_nonzero(const char *str) return value == 0 ? 1 : value; } +static inline uint32_t +strmap_hash_str(const struct mail_index_strmap_view *view, const char *str) +{ + if (view->format_version < MAIL_INDEX_STRMAP_VERSION_V2) + return crc32_str_nonzero(str); + + uint32_t value = xxh64_to_32(xxh64_data(str, strlen(str), view->hash_iv)); + /* 0 is reserved as a sentinel meaning "unique / no hash stored" */ + return value == 0 ? 1 : value; +} + void mail_index_strmap_view_sync_add(struct mail_index_strmap_view_sync *sync, uint32_t uid, uint32_t ref_index, const char *key) @@ -764,7 +833,7 @@ void mail_index_strmap_view_sync_add(struct mail_index_strmap_view_sync *sync, ref_index > view->last_ref_index)); hash_key.str = key; - hash_key.hash = crc32_str_nonzero(key); + hash_key.hash = strmap_hash_str(view, key); old_rec = hash2_lookup(view->hash, &hash_key); if (old_rec != NULL) { @@ -971,14 +1040,29 @@ mail_index_strmap_recreate_write(struct mail_index_strmap_view *view, { const struct mail_index_header *idx_hdr; struct mail_index_strmap_header hdr; + size_t hdr_size; idx_hdr = mail_index_get_header(view->view); + /* view->format_version was set either at view_open() time (from the + strmap-wide enable_xxh64 preference for fresh files) or in + mail_index_strmap_open() from the existing file's header. Reuse + it as-is so renumber recreates preserve the file's format. */ + /* write header */ i_zero(&hdr); - hdr.version = MAIL_INDEX_STRMAP_VERSION; + hdr.version = view->format_version; hdr.uid_validity = idx_hdr->uid_validity; - o_stream_nsend(output, &hdr, sizeof(hdr)); + if (view->format_version >= MAIL_INDEX_STRMAP_VERSION_V2) { +#ifndef WORDS_BIGENDIAN + hdr.compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN; +#endif + hdr.hash_iv = view->hash_iv; + hdr_size = MAIL_INDEX_STRMAP_HEADER_V2_SIZE; + } else { + hdr_size = MAIL_INDEX_STRMAP_HEADER_V1_SIZE; + } + o_stream_nsend(output, &hdr, hdr_size); view->total_ref_count = 0; mail_index_strmap_write_block(view, output, 0, 1); diff --git a/src/lib-index/mail-index-strmap.h b/src/lib-index/mail-index-strmap.h index c61afa6..7d29079 100644 --- a/src/lib-index/mail-index-strmap.h +++ b/src/lib-index/mail-index-strmap.h @@ -7,12 +7,22 @@ struct mail_index; struct mail_index_view; struct mail_index_strmap_header { -#define MAIL_INDEX_STRMAP_VERSION 1 +/* On-disk format version. The header struct below describes version 2; + version 1 files use the same first 8 bytes (version + 3 unused + uid_validity) + without the trailing compat_flags / unused / hash_iv fields. */ +#define MAIL_INDEX_STRMAP_VERSION_V1 1 +#define MAIL_INDEX_STRMAP_VERSION_V2 2 uint8_t version; - uint8_t unused[3]; + uint8_t compat_flags; /* enum mail_index_header_compat_flags, v2+ */ + uint8_t unused[2]; uint32_t uid_validity; + + /* v2+ fields - not present on disk for v1 files. */ + uint64_t hash_iv; /* per-file random IV mixed into the xxh64 hash */ }; +#define MAIL_INDEX_STRMAP_HEADER_V1_SIZE 8 +#define MAIL_INDEX_STRMAP_HEADER_V2_SIZE sizeof(struct mail_index_strmap_header) struct mail_index_strmap_rec { uint32_t uid; @@ -40,7 +50,8 @@ typedef void mail_index_strmap_remap_t(const uint32_t *idx_map, unsigned int new_count, void *context); struct mail_index_strmap * -mail_index_strmap_init(struct mail_index *index, const char *suffix); +mail_index_strmap_init(struct mail_index *index, const char *suffix, + bool enable_xxh64); void mail_index_strmap_deinit(struct mail_index_strmap **strmap); /* Returns strmap records and hash that can be used for read-only access. diff --git a/src/lib-master/Makefile.am b/src/lib-master/Makefile.am index f4b22d1..d6fe251 100644 --- a/src/lib-master/Makefile.am +++ b/src/lib-master/Makefile.am @@ -39,6 +39,7 @@ headers = \ master-service-ssl.h \ service-settings.h \ stats-client.h \ + storage-version.h \ syslog-util.h pkginc_libdir=$(pkgincludedir) diff --git a/src/lib-master/storage-version.h b/src/lib-master/storage-version.h new file mode 100644 index 0000000..f38681e --- /dev/null +++ b/src/lib-master/storage-version.h @@ -0,0 +1,22 @@ +#ifndef STORAGE_VERSION_H +#define STORAGE_VERSION_H + +#include "version.h" + +/* Returns TRUE if dovecot_storage_version selects the v2 (keyed xxh64) + mail_index_strmap on-disk format. */ +static inline bool +storage_version_has_mail_index_strmap_v2(const char *version) +{ + if (version == NULL) { + /* unit test */ + return TRUE; + } +#ifdef DOVECOT_PRO_EDITION + return version_cmp(version, "3.1.6") >= 0; +#else + return version_cmp(version, "2.4.0") >= 0; +#endif +} + +#endif diff --git a/src/lib-storage/index/index-thread.c b/src/lib-storage/index/index-thread.c index a58f968..2a02fd1 100644 --- a/src/lib-storage/index/index-thread.c +++ b/src/lib-storage/index/index-thread.c @@ -7,6 +7,9 @@ #include "bsearch-insert-pos.h" #include "hash2.h" #include "message-id.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "storage-version.h" #include "mail-search.h" #include "mail-search-build.h" #include "mailbox-search-result-private.h" @@ -668,8 +671,15 @@ void index_thread_mailbox_opened(struct mailbox *box) box->v.close = mail_thread_mailbox_close; box->v.free = mail_thread_mailbox_free; + const struct master_service_settings *master_set = + master_service_get_service_settings(master_service); + bool use_xxh64 = master_set != NULL && + storage_version_has_mail_index_strmap_v2( + master_set->dovecot_storage_version); + tbox->strmap = mail_index_strmap_init(box->index, - MAIL_THREAD_INDEX_SUFFIX); + MAIL_THREAD_INDEX_SUFFIX, + use_xxh64); tbox->next_msgid_idx = 1; tbox->cache = i_new(struct mail_thread_cache, 1);