From 9b4164a748baf0015963b4e8901ecc54982cd262 Mon Sep 17 00:00:00 2001 From: Jeremy Sowden Date: Mon, 3 Jun 2024 22:05:55 +0100 Subject: [PATCH] Import maildir-utils_1.12.5.orig.tar.gz [dgit import orig maildir-utils_1.12.5.orig.tar.gz] --- .dir-locals.el | 26 + .editorconfig | 34 + .github/ISSUE_TEMPLATE/feature-request.md | 22 + .github/ISSUE_TEMPLATE/guile.md | 20 + .github/ISSUE_TEMPLATE/misc.md | 16 + .github/ISSUE_TEMPLATE/mu-bug-report.md | 20 + .github/ISSUE_TEMPLATE/mu4e-bug-report.md | 41 + .github/issue_template.md | 39 + .github/workflows/build-and-test.yml | 39 + .gitignore | 141 + .mailmap | 1 + AUTHORS | 1 + COPYING | 674 + IDEAS.org | 49 + Makefile | 158 + NEWS.org | 1582 +++ README.org | 104 + autogen.sh | 26 + build-aux/date.py | 15 + build-aux/meson-install-info.sh | 13 + build-aux/version.texi.in | 5 + contrib/mu-completion.zsh | 124 + contrib/mu-sexp-convert | 204 + contrib/mu.spec | 129 + guile/compile-scm.in | 22 + guile/examples/contacts-export | 85 + guile/examples/msg-graphs | 133 + guile/examples/mu-biff | 59 + guile/examples/org2mu4e | 78 + guile/fdl.texi | 451 + guile/meson.build | 114 + guile/mu-guile-message.cc | 485 + guile/mu-guile-message.hh | 34 + guile/mu-guile-message.x | 6 + guile/mu-guile.cc | 250 + guile/mu-guile.hh | 85 + guile/mu-guile.texi | 995 ++ guile/mu-guile.x | 4 + guile/mu.scm | 318 + guile/mu/README | 207 + guile/mu/contact.scm | 4 + guile/mu/message.scm | 4 + guile/mu/part.scm | 4 + guile/mu/plot.scm | 83 + guile/mu/script.scm | 58 + guile/mu/stats.scm | 167 + guile/scripts/find-dups.scm | 119 + guile/scripts/histogram.scm | 127 + guile/scripts/msgs-count.scm | 40 + guile/tests/meson.build | 38 + guile/tests/test-mu-guile.cc | 130 + guile/tests/test-mu-guile.scm | 124 + lib/meson.build | 95 + lib/message/meson.build | 47 + lib/message/mu-contact.cc | 207 + lib/message/mu-contact.hh | 219 + lib/message/mu-document.cc | 497 + lib/message/mu-document.hh | 261 + lib/message/mu-fields.cc | 194 + lib/message/mu-fields.hh | 605 + lib/message/mu-flags.cc | 196 + lib/message/mu-flags.hh | 422 + lib/message/mu-message-file.cc | 198 + lib/message/mu-message-file.hh | 98 + lib/message/mu-message-part.cc | 256 + lib/message/mu-message-part.hh | 167 + lib/message/mu-message.cc | 863 ++ lib/message/mu-message.hh | 478 + lib/message/mu-mime-object.cc | 798 ++ lib/message/mu-mime-object.hh | 1389 ++ lib/message/mu-priority.cc | 76 + lib/message/mu-priority.hh | 154 + lib/message/test-mu-message.cc | 1125 ++ lib/message/tests/meson.build | 74 + lib/mu-config.cc | 126 + lib/mu-config.hh | 316 + lib/mu-contacts-cache.cc | 609 + lib/mu-contacts-cache.hh | 171 + lib/mu-indexer.cc | 663 + lib/mu-indexer.hh | 122 + lib/mu-maildir.cc | 455 + lib/mu-maildir.hh | 120 + lib/mu-query-macros.cc | 160 + lib/mu-query-macros.hh | 75 + lib/mu-query-match-deciders.cc | 223 + lib/mu-query-match-deciders.hh | 76 + lib/mu-query-parser.cc | 485 + lib/mu-query-parser.hh | 114 + lib/mu-query-processor.cc | 527 + lib/mu-query-results.hh | 422 + lib/mu-query-threads.cc | 957 ++ lib/mu-query-threads.hh | 41 + lib/mu-query-xapianizer.cc | 521 + lib/mu-query.cc | 303 + lib/mu-query.hh | 100 + lib/mu-scanner.cc | 425 + lib/mu-scanner.hh | 122 + lib/mu-script.cc | 162 + lib/mu-script.hh | 65 + lib/mu-server.cc | 1087 ++ lib/mu-server.hh | 89 + lib/mu-store.cc | 673 + lib/mu-store.hh | 490 + lib/mu-xapian-db.cc | 148 + lib/mu-xapian-db.hh | 575 + lib/tests/bench-indexer.cc | 553 + lib/tests/meson.build | 147 + lib/tests/test-mu-container.cc | 80 + lib/tests/test-mu-maildir.cc | 557 + lib/tests/test-mu-msg-fields.cc | 126 + lib/tests/test-mu-msg.cc | 355 + lib/tests/test-mu-store-query.cc | 913 ++ lib/tests/test-mu-store.cc | 590 + lib/tests/test-query.cc | 99 + lib/utils/meson.build | 66 + lib/utils/mu-async-queue.hh | 184 + lib/utils/mu-command-handler.cc | 273 + lib/utils/mu-command-handler.hh | 298 + lib/utils/mu-error.cc | 64 + lib/utils/mu-error.hh | 200 + lib/utils/mu-html-to-text.cc | 598 + lib/utils/mu-lang-detector.cc | 100 + lib/utils/mu-lang-detector.hh | 46 + lib/utils/mu-logger.cc | 239 + lib/utils/mu-logger.hh | 74 + lib/utils/mu-option.cc | 106 + lib/utils/mu-option.hh | 77 + lib/utils/mu-readline.cc | 136 + lib/utils/mu-readline.hh | 61 + lib/utils/mu-regex.cc | 114 + lib/utils/mu-regex.hh | 193 + lib/utils/mu-result.hh | 145 + lib/utils/mu-sexp.cc | 522 + lib/utils/mu-sexp.hh | 326 + lib/utils/mu-test-utils.cc | 140 + lib/utils/mu-test-utils.hh | 167 + lib/utils/mu-unbroken.hh | 127 + lib/utils/mu-utils-file.cc | 521 + lib/utils/mu-utils-file.hh | 275 + lib/utils/mu-utils.cc | 713 + lib/utils/mu-utils.hh | 640 + lib/utils/tests/meson.build | 83 + lib/utils/tests/test-utils.cc | 343 + man/author.inc | 7 + man/bugs.inc | 7 + man/common-options.inc | 31 + man/copyright.inc.in | 12 + man/exit-code.inc | 14 + man/meson.build | 102 + man/mu-add.1.org | 29 + man/mu-bookmarks.5.org | 36 + man/mu-cfind.1.org | 161 + man/mu-easy.7.org | 303 + man/mu-extract.1.org | 108 + man/mu-find.1.org | 314 + man/mu-help.1.org | 20 + man/mu-index.1.org | 218 + man/mu-info.1.org | 32 + man/mu-init.1.org | 100 + man/mu-mkdir.1.org | 42 + man/mu-move.1.org | 117 + man/mu-query.7.org | 384 + man/mu-remove.1.org | 27 + man/mu-server.1.org | 91 + man/mu-verify.1.org | 55 + man/mu-view.1.org | 54 + man/mu.1.org | 90 + man/muhome.inc | 12 + man/prefooter.inc | 9 + meson.build | 306 + meson_options.txt | 49 + mu/meson.build | 43 + mu/mu-cmd-add.cc | 125 + mu/mu-cmd-cfind.cc | 535 + mu/mu-cmd-extract.cc | 306 + mu/mu-cmd-find.cc | 733 ++ mu/mu-cmd-index.cc | 201 + mu/mu-cmd-info.cc | 316 + mu/mu-cmd-init.cc | 144 + mu/mu-cmd-mkdir.cc | 99 + mu/mu-cmd-move.cc | 273 + mu/mu-cmd-remove.cc | 100 + mu/mu-cmd-script.cc | 49 + mu/mu-cmd-server.cc | 164 + mu/mu-cmd-verify.cc | 255 + mu/mu-cmd-view.cc | 424 + mu/mu-cmd.cc | 169 + mu/mu-cmd.hh | 198 + mu/mu-memcheck.in | 6 + mu/mu-options.cc | 956 ++ mu/mu-options.hh | 310 + mu/mu.cc | 136 + mu/tests/gmime-test.c | 264 + mu/tests/meson.build | 110 + mu/tests/test-mu-query.cc | 612 + mu4e/fdl.texi | 451 + mu4e/htmlxref.cnf | 788 ++ mu4e/meson.build | 142 + mu4e/mu4e-about.org | 15 + mu4e/mu4e-actions.el | 275 + mu4e/mu4e-bookmarks.el | 195 + mu4e/mu4e-compose.el | 521 + mu4e/mu4e-config.el.in | 9 + mu4e/mu4e-contacts.el | 308 + mu4e/mu4e-context.el | 243 + mu4e/mu4e-contrib.el | 201 + mu4e/mu4e-draft.el | 746 ++ mu4e/mu4e-folders.el | 302 + mu4e/mu4e-headers.el | 1648 +++ mu4e/mu4e-helpers.el | 604 + mu4e/mu4e-icalendar.el | 210 + mu4e/mu4e-lists.el | 170 + mu4e/mu4e-main.el | 435 + mu4e/mu4e-mark.el | 471 + mu4e/mu4e-message.el | 247 + mu4e/mu4e-mime-parts.el | 486 + mu4e/mu4e-modeline.el | 141 + mu4e/mu4e-notification.el | 99 + mu4e/mu4e-obsolete.el | 283 + mu4e/mu4e-org.el | 146 + mu4e/mu4e-pkg.el.in | 7 + mu4e/mu4e-query-items.el | 254 + mu4e/mu4e-search.el | 635 + mu4e/mu4e-server.el | 712 + mu4e/mu4e-speedbar.el | 133 + mu4e/mu4e-thread.el | 295 + mu4e/mu4e-update.el | 335 + mu4e/mu4e-vars.el | 392 + mu4e/mu4e-view.el | 1167 ++ mu4e/mu4e-window.el | 383 + mu4e/mu4e.el | 266 + mu4e/mu4e.texi | 4948 +++++++ mu4e/texinfo-klare.css | 228 + testdata/cjk/cur/test1 | 10 + testdata/cjk/cur/test2 | 10 + testdata/cjk/cur/test3 | 10 + testdata/cjk/cur/test4 | 10 + .../cur/1220863042.12663_1.mindcrime!2,S | 146 + .../cur/1220863060.12663_3.mindcrime!2,S | 230 + .../cur/1220863087.12663_15.mindcrime!2,PS | 136 + .../cur/1220863087.12663_19.mindcrime!2,S | 77 + .../cur/1220863087.12663_5.mindcrime!2,S | 84 + .../cur/1220863087.12663_7.mindcrime!2,RS | 138 + .../cur/1252168370_3.14675.cthulhu!2,S | 21 + .../testdir/cur/1283599333.1840_11.cthulhu!2, | 16 + .../cur/1305664394.2171_402.cthulhu!2, | 17 + testdata/testdir/cur/encrypted!2,S | 56 + testdata/testdir/cur/multimime!2,FS | 27 + testdata/testdir/cur/multirecip!2,S | 11 + testdata/testdir/cur/signed!2,S | 36 + testdata/testdir/cur/signed-encrypted!2,S | 54 + testdata/testdir/cur/special!2,Sabc | 10 + .../testdir/new/1220863087.12663_21.mindcrime | 111 + .../testdir/new/1220863087.12663_23.mindcrime | 105 + .../testdir/new/1220863087.12663_25.mindcrime | 98 + .../testdir/new/1220863087.12663_9.mindcrime | 209 + testdata/testdir/tmp/1220863087.12663.ignore | 98 + testdata/testdir2/Foo/cur/arto.eml | 448 + testdata/testdir2/Foo/cur/fraiche.eml | 10 + testdata/testdir2/Foo/cur/mail5 | 625 + testdata/testdir2/Foo/new/.noindex | 0 testdata/testdir2/Foo/tmp/.noindex | 0 testdata/testdir2/bar/.noupdate | 0 testdata/testdir2/bar/cur/181736.eml | 42 + testdata/testdir2/bar/cur/mail1 | 38 + testdata/testdir2/bar/cur/mail2 | 14 + testdata/testdir2/bar/cur/mail3 | 34 + testdata/testdir2/bar/cur/mail4 | 29 + testdata/testdir2/bar/cur/mail5 | 7 + testdata/testdir2/bar/cur/mail6 | 18 + testdata/testdir2/bar/cur/mail7 | 16 + testdata/testdir2/bar/new/.noindex | 0 testdata/testdir2/bar/tmp/.noindex | 0 testdata/testdir2/wom_bat/cur/atomic | 20 + testdata/testdir2/wom_bat/cur/rfc822.1 | 44 + testdata/testdir2/wom_bat/cur/rfc822.2 | 44 + .../testdir4/1220863042.12663_1.mindcrime!2,S | 146 + .../1220863087.12663_19.mindcrime!2,S | 77 + .../testdir4/1252168370_3.14675.cthulhu!2,S | 22 + .../testdir4/1283599333.1840_11.cthulhu!2, | 15 + .../testdir4/1305664394.2171_402.cthulhu!2, | 17 + testdata/testdir4/181736.eml | 42 + testdata/testdir4/encrypted!2,S | 57 + testdata/testdir4/mail1 | 38 + testdata/testdir4/mail5 | 624 + testdata/testdir4/multimime!2,FS | 27 + testdata/testdir4/signed!2,S | 36 + testdata/testdir4/signed-bad!2,S | 35 + testdata/testdir4/signed-encrypted!2,S | 54 + testdata/testdir4/special!2,Sabc | 10 + thirdparty/CLI11.hpp | 10966 ++++++++++++++++ thirdparty/fmt/LICENSE.rst | 27 + thirdparty/fmt/args.h | 235 + thirdparty/fmt/chrono.h | 2240 ++++ thirdparty/fmt/color.h | 643 + thirdparty/fmt/compile.h | 535 + thirdparty/fmt/core.h | 2969 +++++ thirdparty/fmt/format-inl.h | 1678 +++ thirdparty/fmt/format.h | 4535 +++++++ thirdparty/fmt/os.h | 455 + thirdparty/fmt/ostream.h | 245 + thirdparty/fmt/printf.h | 675 + thirdparty/fmt/ranges.h | 738 ++ thirdparty/fmt/std.h | 537 + thirdparty/fmt/xchar.h | 259 + thirdparty/tabulate.hpp | 9255 +++++++++++++ thirdparty/tl/expected.hpp | 2444 ++++ thirdparty/tl/optional.hpp | 2062 +++ 308 files changed, 109755 insertions(+) create mode 100644 .dir-locals.el create mode 100644 .editorconfig create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md create mode 100644 .github/ISSUE_TEMPLATE/guile.md create mode 100644 .github/ISSUE_TEMPLATE/misc.md create mode 100644 .github/ISSUE_TEMPLATE/mu-bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/mu4e-bug-report.md create mode 100644 .github/issue_template.md create mode 100644 .github/workflows/build-and-test.yml create mode 100644 .gitignore create mode 100644 .mailmap create mode 100644 AUTHORS create mode 100644 COPYING create mode 100644 IDEAS.org create mode 100644 Makefile create mode 100644 NEWS.org create mode 100644 README.org create mode 100755 autogen.sh create mode 100755 build-aux/date.py create mode 100644 build-aux/meson-install-info.sh create mode 100644 build-aux/version.texi.in create mode 100644 contrib/mu-completion.zsh create mode 100755 contrib/mu-sexp-convert create mode 100644 contrib/mu.spec create mode 100644 guile/compile-scm.in create mode 100755 guile/examples/contacts-export create mode 100755 guile/examples/msg-graphs create mode 100755 guile/examples/mu-biff create mode 100755 guile/examples/org2mu4e create mode 100644 guile/fdl.texi create mode 100644 guile/meson.build create mode 100644 guile/mu-guile-message.cc create mode 100644 guile/mu-guile-message.hh create mode 100644 guile/mu-guile-message.x create mode 100644 guile/mu-guile.cc create mode 100644 guile/mu-guile.hh create mode 100644 guile/mu-guile.texi create mode 100644 guile/mu-guile.x create mode 100644 guile/mu.scm create mode 100644 guile/mu/README create mode 100644 guile/mu/contact.scm create mode 100644 guile/mu/message.scm create mode 100644 guile/mu/part.scm create mode 100644 guile/mu/plot.scm create mode 100644 guile/mu/script.scm create mode 100644 guile/mu/stats.scm create mode 100755 guile/scripts/find-dups.scm create mode 100755 guile/scripts/histogram.scm create mode 100755 guile/scripts/msgs-count.scm create mode 100644 guile/tests/meson.build create mode 100644 guile/tests/test-mu-guile.cc create mode 100755 guile/tests/test-mu-guile.scm create mode 100644 lib/meson.build create mode 100644 lib/message/meson.build create mode 100644 lib/message/mu-contact.cc create mode 100644 lib/message/mu-contact.hh create mode 100644 lib/message/mu-document.cc create mode 100644 lib/message/mu-document.hh create mode 100644 lib/message/mu-fields.cc create mode 100644 lib/message/mu-fields.hh create mode 100644 lib/message/mu-flags.cc create mode 100644 lib/message/mu-flags.hh create mode 100644 lib/message/mu-message-file.cc create mode 100644 lib/message/mu-message-file.hh create mode 100644 lib/message/mu-message-part.cc create mode 100644 lib/message/mu-message-part.hh create mode 100644 lib/message/mu-message.cc create mode 100644 lib/message/mu-message.hh create mode 100644 lib/message/mu-mime-object.cc create mode 100644 lib/message/mu-mime-object.hh create mode 100644 lib/message/mu-priority.cc create mode 100644 lib/message/mu-priority.hh create mode 100644 lib/message/test-mu-message.cc create mode 100644 lib/message/tests/meson.build create mode 100644 lib/mu-config.cc create mode 100644 lib/mu-config.hh create mode 100644 lib/mu-contacts-cache.cc create mode 100644 lib/mu-contacts-cache.hh create mode 100644 lib/mu-indexer.cc create mode 100644 lib/mu-indexer.hh create mode 100644 lib/mu-maildir.cc create mode 100644 lib/mu-maildir.hh create mode 100644 lib/mu-query-macros.cc create mode 100644 lib/mu-query-macros.hh create mode 100644 lib/mu-query-match-deciders.cc create mode 100644 lib/mu-query-match-deciders.hh create mode 100644 lib/mu-query-parser.cc create mode 100644 lib/mu-query-parser.hh create mode 100644 lib/mu-query-processor.cc create mode 100644 lib/mu-query-results.hh create mode 100644 lib/mu-query-threads.cc create mode 100644 lib/mu-query-threads.hh create mode 100644 lib/mu-query-xapianizer.cc create mode 100644 lib/mu-query.cc create mode 100644 lib/mu-query.hh create mode 100644 lib/mu-scanner.cc create mode 100644 lib/mu-scanner.hh create mode 100644 lib/mu-script.cc create mode 100644 lib/mu-script.hh create mode 100644 lib/mu-server.cc create mode 100644 lib/mu-server.hh create mode 100644 lib/mu-store.cc create mode 100644 lib/mu-store.hh create mode 100644 lib/mu-xapian-db.cc create mode 100644 lib/mu-xapian-db.hh create mode 100644 lib/tests/bench-indexer.cc create mode 100644 lib/tests/meson.build create mode 100644 lib/tests/test-mu-container.cc create mode 100644 lib/tests/test-mu-maildir.cc create mode 100644 lib/tests/test-mu-msg-fields.cc create mode 100644 lib/tests/test-mu-msg.cc create mode 100644 lib/tests/test-mu-store-query.cc create mode 100644 lib/tests/test-mu-store.cc create mode 100644 lib/tests/test-query.cc create mode 100644 lib/utils/meson.build create mode 100644 lib/utils/mu-async-queue.hh create mode 100644 lib/utils/mu-command-handler.cc create mode 100644 lib/utils/mu-command-handler.hh create mode 100644 lib/utils/mu-error.cc create mode 100644 lib/utils/mu-error.hh create mode 100644 lib/utils/mu-html-to-text.cc create mode 100644 lib/utils/mu-lang-detector.cc create mode 100644 lib/utils/mu-lang-detector.hh create mode 100644 lib/utils/mu-logger.cc create mode 100644 lib/utils/mu-logger.hh create mode 100644 lib/utils/mu-option.cc create mode 100644 lib/utils/mu-option.hh create mode 100644 lib/utils/mu-readline.cc create mode 100644 lib/utils/mu-readline.hh create mode 100644 lib/utils/mu-regex.cc create mode 100644 lib/utils/mu-regex.hh create mode 100644 lib/utils/mu-result.hh create mode 100644 lib/utils/mu-sexp.cc create mode 100644 lib/utils/mu-sexp.hh create mode 100644 lib/utils/mu-test-utils.cc create mode 100644 lib/utils/mu-test-utils.hh create mode 100644 lib/utils/mu-unbroken.hh create mode 100644 lib/utils/mu-utils-file.cc create mode 100644 lib/utils/mu-utils-file.hh create mode 100644 lib/utils/mu-utils.cc create mode 100644 lib/utils/mu-utils.hh create mode 100644 lib/utils/tests/meson.build create mode 100644 lib/utils/tests/test-utils.cc create mode 100644 man/author.inc create mode 100644 man/bugs.inc create mode 100644 man/common-options.inc create mode 100644 man/copyright.inc.in create mode 100644 man/exit-code.inc create mode 100644 man/meson.build create mode 100644 man/mu-add.1.org create mode 100644 man/mu-bookmarks.5.org create mode 100644 man/mu-cfind.1.org create mode 100644 man/mu-easy.7.org create mode 100644 man/mu-extract.1.org create mode 100644 man/mu-find.1.org create mode 100644 man/mu-help.1.org create mode 100644 man/mu-index.1.org create mode 100644 man/mu-info.1.org create mode 100644 man/mu-init.1.org create mode 100644 man/mu-mkdir.1.org create mode 100644 man/mu-move.1.org create mode 100644 man/mu-query.7.org create mode 100644 man/mu-remove.1.org create mode 100644 man/mu-server.1.org create mode 100644 man/mu-verify.1.org create mode 100644 man/mu-view.1.org create mode 100644 man/mu.1.org create mode 100644 man/muhome.inc create mode 100644 man/prefooter.inc create mode 100644 meson.build create mode 100644 meson_options.txt create mode 100644 mu/meson.build create mode 100644 mu/mu-cmd-add.cc create mode 100644 mu/mu-cmd-cfind.cc create mode 100644 mu/mu-cmd-extract.cc create mode 100644 mu/mu-cmd-find.cc create mode 100644 mu/mu-cmd-index.cc create mode 100644 mu/mu-cmd-info.cc create mode 100644 mu/mu-cmd-init.cc create mode 100644 mu/mu-cmd-mkdir.cc create mode 100644 mu/mu-cmd-move.cc create mode 100644 mu/mu-cmd-remove.cc create mode 100644 mu/mu-cmd-script.cc create mode 100644 mu/mu-cmd-server.cc create mode 100644 mu/mu-cmd-verify.cc create mode 100644 mu/mu-cmd-view.cc create mode 100644 mu/mu-cmd.cc create mode 100644 mu/mu-cmd.hh create mode 100644 mu/mu-memcheck.in create mode 100644 mu/mu-options.cc create mode 100644 mu/mu-options.hh create mode 100644 mu/mu.cc create mode 100644 mu/tests/gmime-test.c create mode 100644 mu/tests/meson.build create mode 100644 mu/tests/test-mu-query.cc create mode 100644 mu4e/fdl.texi create mode 100644 mu4e/htmlxref.cnf create mode 100644 mu4e/meson.build create mode 100644 mu4e/mu4e-about.org create mode 100644 mu4e/mu4e-actions.el create mode 100644 mu4e/mu4e-bookmarks.el create mode 100644 mu4e/mu4e-compose.el create mode 100644 mu4e/mu4e-config.el.in create mode 100644 mu4e/mu4e-contacts.el create mode 100644 mu4e/mu4e-context.el create mode 100644 mu4e/mu4e-contrib.el create mode 100644 mu4e/mu4e-draft.el create mode 100644 mu4e/mu4e-folders.el create mode 100644 mu4e/mu4e-headers.el create mode 100644 mu4e/mu4e-helpers.el create mode 100644 mu4e/mu4e-icalendar.el create mode 100644 mu4e/mu4e-lists.el create mode 100644 mu4e/mu4e-main.el create mode 100644 mu4e/mu4e-mark.el create mode 100644 mu4e/mu4e-message.el create mode 100644 mu4e/mu4e-mime-parts.el create mode 100644 mu4e/mu4e-modeline.el create mode 100644 mu4e/mu4e-notification.el create mode 100644 mu4e/mu4e-obsolete.el create mode 100644 mu4e/mu4e-org.el create mode 100644 mu4e/mu4e-pkg.el.in create mode 100644 mu4e/mu4e-query-items.el create mode 100644 mu4e/mu4e-search.el create mode 100644 mu4e/mu4e-server.el create mode 100644 mu4e/mu4e-speedbar.el create mode 100644 mu4e/mu4e-thread.el create mode 100644 mu4e/mu4e-update.el create mode 100644 mu4e/mu4e-vars.el create mode 100644 mu4e/mu4e-view.el create mode 100644 mu4e/mu4e-window.el create mode 100644 mu4e/mu4e.el create mode 100644 mu4e/mu4e.texi create mode 100644 mu4e/texinfo-klare.css create mode 100644 testdata/cjk/cur/test1 create mode 100644 testdata/cjk/cur/test2 create mode 100644 testdata/cjk/cur/test3 create mode 100644 testdata/cjk/cur/test4 create mode 100644 testdata/testdir/cur/1220863042.12663_1.mindcrime!2,S create mode 100644 testdata/testdir/cur/1220863060.12663_3.mindcrime!2,S create mode 100644 testdata/testdir/cur/1220863087.12663_15.mindcrime!2,PS create mode 100644 testdata/testdir/cur/1220863087.12663_19.mindcrime!2,S create mode 100644 testdata/testdir/cur/1220863087.12663_5.mindcrime!2,S create mode 100644 testdata/testdir/cur/1220863087.12663_7.mindcrime!2,RS create mode 100644 testdata/testdir/cur/1252168370_3.14675.cthulhu!2,S create mode 100644 testdata/testdir/cur/1283599333.1840_11.cthulhu!2, create mode 100644 testdata/testdir/cur/1305664394.2171_402.cthulhu!2, create mode 100644 testdata/testdir/cur/encrypted!2,S create mode 100644 testdata/testdir/cur/multimime!2,FS create mode 100644 testdata/testdir/cur/multirecip!2,S create mode 100644 testdata/testdir/cur/signed!2,S create mode 100644 testdata/testdir/cur/signed-encrypted!2,S create mode 100644 testdata/testdir/cur/special!2,Sabc create mode 100644 testdata/testdir/new/1220863087.12663_21.mindcrime create mode 100644 testdata/testdir/new/1220863087.12663_23.mindcrime create mode 100644 testdata/testdir/new/1220863087.12663_25.mindcrime create mode 100644 testdata/testdir/new/1220863087.12663_9.mindcrime create mode 100644 testdata/testdir/tmp/1220863087.12663.ignore create mode 100644 testdata/testdir2/Foo/cur/arto.eml create mode 100644 testdata/testdir2/Foo/cur/fraiche.eml create mode 100644 testdata/testdir2/Foo/cur/mail5 create mode 100644 testdata/testdir2/Foo/new/.noindex create mode 100644 testdata/testdir2/Foo/tmp/.noindex create mode 100644 testdata/testdir2/bar/.noupdate create mode 100644 testdata/testdir2/bar/cur/181736.eml create mode 100644 testdata/testdir2/bar/cur/mail1 create mode 100644 testdata/testdir2/bar/cur/mail2 create mode 100644 testdata/testdir2/bar/cur/mail3 create mode 100644 testdata/testdir2/bar/cur/mail4 create mode 100644 testdata/testdir2/bar/cur/mail5 create mode 100644 testdata/testdir2/bar/cur/mail6 create mode 100644 testdata/testdir2/bar/cur/mail7 create mode 100644 testdata/testdir2/bar/new/.noindex create mode 100644 testdata/testdir2/bar/tmp/.noindex create mode 100644 testdata/testdir2/wom_bat/cur/atomic create mode 100644 testdata/testdir2/wom_bat/cur/rfc822.1 create mode 100644 testdata/testdir2/wom_bat/cur/rfc822.2 create mode 100644 testdata/testdir4/1220863042.12663_1.mindcrime!2,S create mode 100644 testdata/testdir4/1220863087.12663_19.mindcrime!2,S create mode 100644 testdata/testdir4/1252168370_3.14675.cthulhu!2,S create mode 100644 testdata/testdir4/1283599333.1840_11.cthulhu!2, create mode 100644 testdata/testdir4/1305664394.2171_402.cthulhu!2, create mode 100644 testdata/testdir4/181736.eml create mode 100644 testdata/testdir4/encrypted!2,S create mode 100644 testdata/testdir4/mail1 create mode 100644 testdata/testdir4/mail5 create mode 100644 testdata/testdir4/multimime!2,FS create mode 100644 testdata/testdir4/signed!2,S create mode 100644 testdata/testdir4/signed-bad!2,S create mode 100644 testdata/testdir4/signed-encrypted!2,S create mode 100644 testdata/testdir4/special!2,Sabc create mode 100644 thirdparty/CLI11.hpp create mode 100644 thirdparty/fmt/LICENSE.rst create mode 100644 thirdparty/fmt/args.h create mode 100644 thirdparty/fmt/chrono.h create mode 100644 thirdparty/fmt/color.h create mode 100644 thirdparty/fmt/compile.h create mode 100644 thirdparty/fmt/core.h create mode 100644 thirdparty/fmt/format-inl.h create mode 100644 thirdparty/fmt/format.h create mode 100644 thirdparty/fmt/os.h create mode 100644 thirdparty/fmt/ostream.h create mode 100644 thirdparty/fmt/printf.h create mode 100644 thirdparty/fmt/ranges.h create mode 100644 thirdparty/fmt/std.h create mode 100644 thirdparty/fmt/xchar.h create mode 100644 thirdparty/tabulate.hpp create mode 100644 thirdparty/tl/expected.hpp create mode 100644 thirdparty/tl/optional.hpp diff --git a/.dir-locals.el b/.dir-locals.el new file mode 100644 index 0000000..997a80e --- /dev/null +++ b/.dir-locals.el @@ -0,0 +1,26 @@ +;;; -*- no-byte-compile: t; -*- +((nil . ((tab-width . 8) + (fill-column . 80) + ;; (commment-fill-column . 80) + (emacs-lisp-docstring-fill-column . 65) + (bug-reference-url-format . "https://github.com/djcb/mu/issues/%s"))) + (c-mode . ((c-file-style . "linux") + (indent-tabs-mode . t) + (mode . bug-reference-prog))) + (c-ts-mode . ((indent-tabs-mode . t) + (c-ts-mode-indent-style . linux) + (c-ts-mode-indent-offset . 8) + (mode . bug-reference-prog))) + (c++-mode . ((c-file-style . "linux") + (fill-column . 100) + ;; (comment-fill-column . 80) + (mode . bug-reference-prog))) + (c++-ts-mode . ((indent-tabs-mode . t) + (c-ts-mode-indent-style . linux) + (c-ts-mode-indent-offset . 8) + (mode . bug-reference-prog))) + (emacs-lisp-mode . ((indent-tabs-mode . nil) + (mode . bug-reference-prog))) + (lisp-data-mode . ((indent-tabs-mode . nil))) + (texinfo-mode . ((mode . bug-reference-prog))) + (org-mode . ((mode . bug-reference)))) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..824f406 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,34 @@ +#-*-mode:conf-*- +# editorconfig file (see EditorConfig.org), with some +# lowest-denominator settings that should work for many editors. + + +root = true # this is the top-level + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true + +# The "best" answer is "tabs-for-indentation; spaces for alignment". + +[*.{cc,cpp,hh,hpp}] +indent_style = tab +indent_size = 8 +max_line_length = 90 + +[*.{c,h}] +indent_style = tab +indent_size = 8 +max_line_length = 80 + +[configure.ac] +indent_style = tab +indent_size = 4 +max_line_length = 100 + +[Makefile.am] +indent_style = tab +indent_size = 8 +max_line_length = 100 diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..77f0195 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,22 @@ +--- +name: Mu4e Feature request +about: Suggest an idea for this project +title: "[mu4e rfe]" +labels: rfe, mu4e, new +assignees: '' + +--- +Note, please see the IDEAS.org file in repository root for existing ideas; +maybe it's already there. + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/guile.md b/.github/ISSUE_TEMPLATE/guile.md new file mode 100644 index 0000000..020b849 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/guile.md @@ -0,0 +1,20 @@ +--- +name: Guile +about: mu-guile related item +title: "[guile]" +labels: new, guile +assignees: '' + +--- + +**Describe the item** +A clear and concise description of what you expected or wished to happen and what actually happened while using mu-guile. + +**To Reproduce** +Steps to reproduce the behavior. + +**Environment** +Please describe the versions of OS, Emacs, mu/mu4e etc. you are using. + +**Checklist** +- [ ] you are running either the latest 1.4.x release, or a 1.5.11+ development release (otherwise, please upgrade). diff --git a/.github/ISSUE_TEMPLATE/misc.md b/.github/ISSUE_TEMPLATE/misc.md new file mode 100644 index 0000000..7f942cd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/misc.md @@ -0,0 +1,16 @@ +--- +name: Misc +about: Miscellaneous items you want to share +title: "[misc]" +labels: new +assignees: '' + +--- + +**Note**: for questions, please use the mailing-list: https://groups.google.com/g/mu-discuss + +**Describe the issue** +A clear and concise description, i.e. what you expected/desired to happen and what actually happened. + +**Environment** +If applicable, please describe the versions of OS, Emacs, mu etc. you are using. diff --git a/.github/ISSUE_TEMPLATE/mu-bug-report.md b/.github/ISSUE_TEMPLATE/mu-bug-report.md new file mode 100644 index 0000000..ef2b3f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/mu-bug-report.md @@ -0,0 +1,20 @@ +--- +name: Mu Bug Report +about: Create a report to help us improve +title: "[mu bug]" +labels: bug, mu, new +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is, what you expected to happen and what actually happened. + +**To Reproduce** +Detailed steps to reproduce the behavior. If this is about a specific (kind of) message, **always** attach an (anonymized as need) example message. + +**Environment** +Please describe the versions of OS, Emacs, mu etc. you are using. + +**Checklist** +- [ ] you are running either the latest 1.8.x/1.10.x release or `master` (otherwise, please upgrade). diff --git a/.github/ISSUE_TEMPLATE/mu4e-bug-report.md b/.github/ISSUE_TEMPLATE/mu4e-bug-report.md new file mode 100644 index 0000000..63d857f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/mu4e-bug-report.md @@ -0,0 +1,41 @@ +--- +name: Mu4e Bug Report +about: Create a report to help us improve +title: "[mu4e bug]" +labels: bug, mu4e, new +assignees: '' +--- + +**Describe the bug** + +Give the bug a good title. + +Please provide a clear and concise description of what you expected to happen +and what actually happened, and follow the steps below. + +**How to Reproduce** + +Include the exact steps of what you were doing (commands executed etc.). Include +any relevant logs and outputs: + +- Best start from `emacs -Q`, and load a minimal `mu4e` setup; describe the steps + that lead up to the bug. +- Does the problem happen each time? Sometimes? +- If this is about a specific (kind of) message, attach an example message. + (Open the message, press `.` (`mu4e-view-raw-message`), then `C-x C-w` and + attach. Anonymize as needed, all that matters is that the issue still + reproduces. + +**Environment** + +Please describe the versions of OS, Emacs, mu/mu4e etc. you are using. + +**Checklist** + +- [ ] you are running either an 1.10.x/1.12.x release or `master` (otherwise please upgrade) +- [ ] you can reproduce the problem without 3rd party extensions (including Doom/Evil, various extensions etc.) +- [ ] you have read all of the above + +Please make sure you all items in the checklist are set/met before filing the ticket. + +Thank you! diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 0000000..a684c39 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,39 @@ +# Important! Before filing an issue, please consider the following: + + * Ensure your mu/mu4e setup is no older than the latest stable release (1.6.x). + + * Disable any third-party mu4e extensions; this includes customizations like the ones in "Doom" / + "Evil" etc. + + * If a problem occurs with a certain (type of) message, attach an (anonymized) example of + such a message + + * Please provide some minimal steps to reproduce + + * Please follow the below template + + Thanks! + +## Expected or desired behavior + +Please describe the behavior you expect or want + +## Actual behavior + +Please describe the behavior you are actually seeing. + +For bug-reports, if applicable, include error messages, emacs stack traces, example messages +etc. Try to be as specific as possible - when do you see this happening? Does it happen always? +Sometimes? How often? + +## Steps to reproduce + +For bug-reports, please describe in as much detail as possible how one can reproduce the problem. + +If there's a problem with a specific (type of) message, please attach such a message to the report. + +## Versions of mu, mu4e/emacs, operating system etc. + +## Any other detail + +E.g. are you using the gnus-based message view? diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000..508901c --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,39 @@ +name: Build & run tests + +on: + - push + - pull_request + +jobs: + build: + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + + steps: + - uses: actions/checkout@v2 + + - if: contains(matrix.os, 'ubuntu') + name: ubuntu-deps + run: | + sudo apt update + sudo apt-get install meson ninja-build libglib2.0-dev libxapian-dev libgmime-3.0-dev libcld2-dev pkg-config guile-3.0-dev emacs texinfo + + - if: contains(matrix.os, 'macos') + name: macos-deps + run: | + brew install meson ninja libgpg-error libtool pkg-config glib gmime xapian guile emacs texinfo + + - name: configure + run: ./autogen.sh # '-Db_sanitize=address' + + - name: build + run: make + + - name: test + run: make test-verbose-if-fail diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d785f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,141 @@ +www/mu +mug +/mu/mu +/mu/mu-help-strings.h +mug2 +.desktop +*html +.deps +.libs +autom4te* +Makefile +Makefile.in +INSTALL +aclocal.m4 +config.* +configure +install-sh +depcomp +libtool +ltmain.sh + +# Added automatically by `autoreconf` +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 + +missing +nohup.out +vgdump +stamp-h1 +GPATH +GRTAGS +GSYMS +GTAGS +*.lo +*.o +*.la +*.x +*.go +*.gz +*.bz2 +\#* +*.aux +*.cp +*.fn +*.info +*.ky +*.log +*.pg +*.toc +*.tp +*.vr +*.elc +*.gcda +*.gcno +*.trs +*.exe +*.lib +aminclude_static.am +elisp-comp +elc-stamp +dummy.cc +msg2pdf +gmime-test +test-mu-cmd +test-mu-cmd-cfind +test-mu-contacts +test-mu-container +test-mu-date +test-mu-flags +test-mu-maildir +test-mu-msg +test-mu-msg-fields +test-mu-query +test-mu-runtime +test-mu-store +test-mu-str +test-mu-threads +test-mu-util +test-parser +test-tokenizer +test-utils +tokenize +test-command-parser +test-mu-utils +test-sexp-parser +test-scanner +/guile/tests/test-mu-guile + +mu4e-config.el +mu4e.pdf +texinfo.tex +texi.texi +*.tex +*.pdf +/www/auto/* +configure.lineno +/test.xml +/mu4e/mdate-sh +/mu4e/mu4e-about.el +/mu4e/stamp-vti +/mu4e/version.texi +/lib/doxyfile +/version.texi +/compile +/TAGS +parse + +*_flymake.* +*_flymake_* +/perf.data +perf.data +perf.data.old +*vgdump +/lib/asan.log* +/man/mu-mfind.1 +/mu/mu-memcheck +mu-*-coverage +mu*tar.xz +compile_commands.json +/lib/utils/test-sexp +/lib/utils/test-option +/lib/test-mu-threader +/lib/test-mu-tokenizer +/lib/test-mu-parser +/lib/test-mu-query-threader +/lib/test-contacts +/lib/test-flags +/lib/test-maildir +/lib/test-msg +/lib/test-msg-fields +/lib/test-query +/lib/test-store +/lib/test-threader +/mu/test-cmd +/mu/test-cmd-cfind +/mu/test-query +/mu/test-threads +/lib/test-threads diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..3a54641 --- /dev/null +++ b/.mailmap @@ -0,0 +1 @@ +Dirk-Jan C. Binnema diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..3a54641 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Dirk-Jan C. Binnema diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/IDEAS.org b/IDEAS.org new file mode 100644 index 0000000..e3e954e --- /dev/null +++ b/IDEAS.org @@ -0,0 +1,49 @@ +#+STARTUP:showall +* IDEAS + +Ideas for future enhancements. We collect those here so they don't clutter up +the Github issue list, i.e. without any clear plan for adding in the near +future. + +- Ability to _mute_ message threads. This is useful but also requires quite bit of extra infra; we could add some blacklist for "muted" messages, perhaps on the 'mu server' side, but then we'd need some way to manage that (ie., unmute). + https://github.com/djcb/mu/issues/636 + +- Support automatic handling for List-Unsubscribe headers + https://github.com/djcb/mu/issues/2623 This seems useful, but probably + requires a lot of testing to get right. + +- Allow for *muting* messages https://github.com/djcb/mu/issues/636 Useful; + probably need to do this by *remembering* the thread-id of muted messages; and + management (unmute etc.). Perhaps at the mu side, a list of thread-id to add + to each query for what *not* to match. + +- Support *creating* calendar invitations. + https://github.com/djcb/mu/issues/2308 + Shouldn't be _too_ hard, for someone that uses the functionality. + +- Make sorting stable if there are multiple messages with the same date. We + _could_ do this by adding some random millisecs to each messasge's timestamp; _or_ + complicating the search (i.e., the message hash?). Maybe leave as is? + https://github.com/djcb/mu/issues/2527 + +- Include "message summary" in message information, for display in the headers + buffer: https://github.com/djcb/mu/issues/1821 It's not so easy to get a + useful one line description... perhaps the first line after the "Dear x,"? + Moreover, this requires new functionality on the headers-view side as well. + +- Support indexing PDF (and other) attachments. This can be done extending + process_message_part in mu-message.cc; instead of using something + PDF-specific, we could pipe a PDF through some tool to extract text; and we'd + need some way for users to specify a MIME-type => tool mapping (in Config). + https://github.com/djcb/mu/issues/2117 + +- Support "aggregate actions" apply to a set of messages, e.g. apply patch-set + in a set of messages. That'll require some advanced scripting, maybe using + Guile. + https://github.com/djcb/mu/issues/301 + +* Done + +- Support mu4e-mark-handle-when also for when leaving emacs + (kill-emacs-query-functions). + https://github.com/djcb/mu/issues/2649 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..662eda7 --- /dev/null +++ b/Makefile @@ -0,0 +1,158 @@ +## Copyright (C) 2008-2023 Dirk-Jan C. Binnema +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Makefile with some useful targets for meson/ninja +V ?= 0 + +BUILDDIR ?= $(CURDIR)/build +BUILDDIR_COVERAGE ?= $(CURDIR)/build-coverage +BUILDDIR_VALGRIND ?= $(CURDIR)/build-valgrind +BUILDDIR_BENCHMARK ?= $(CURDIR)/build-benchmark + +GENHTML ?= genhtml +LCOV ?= lcov +MAKEINFO ?= makeinfo +MESON ?= meson +NINJA ?= ninja +VALGRIND ?= valgrind + +ifneq ($(V),0) + VERBOSE=--verbose +endif + +# when MU_HACKER is set, do a debug build +# MU_HACKER is for djcb & compatible developers +# note that mu uses C++17, we only pass C++23 here +# for the better error messages (esp. for fmt). +ifneq (${MU_HACKER},) +MESON_FLAGS:=$(MESON_FLAGS) '-Dbuildtype=debug' \ + '-Db_sanitize=address' \ + '-Dreadline=enabled' \ + '-Dcpp_std=c++23' +endif + +.PHONY: all build-valgrind +.PHONY: check test test-verbose-if-fail test-valgrind test-helgrind +.PHONY: benchmark coverage +.PHONY: dist install uninstall clean distclean +.PHONY: mu4e-doc-html + +# MESON_FLAGS, e.g. "-Dreadline=enabled" + +# examples: +# 1. build with clang, and the thread-sanitizer +# make clean all MESON_FLAGS="-Db_sanitize=thread" CXX=clang++ CC=clang +all: $(BUILDDIR) + @$(MESON) compile -C $(BUILDDIR) $(VERBOSE) + @ln -sf $(BUILDDIR)/compile_commands.json $(CURDIR) || /bin/true + +$(BUILDDIR): + @$(MESON) setup $(MESON_FLAGS) $(BUILDDIR) + +check: test + +test: all + @$(MESON) test $(VERBOSE) -C $(BUILDDIR) + +install: $(BUILDDIR) + @$(MESON) install -C $(BUILDDIR) $(VERBOSE) + +uninstall: $(BUILDDIR) + @$(NINJA) -C $(BUILDDIR) uninstall + +clean: + @rm -rf $(BUILDDIR) $(BUILDDIR_COVERAGE) $(BUILDDIR_VALGRIND) $(BUILDDIR_BENCHMARK) + @rm -rf compile_commands.json + +# +# below targets are just for development/testing/debugging. They may or +# may not work on your system. +# +test-verbose-if-fail: all + $(MESON) test -C $(BUILDDIR) || $(MESON) test -C $(BUILDDIR) --verbose + +build-valgrind: $(BUILDDIR_VALGRIND) + @$(MESON) compile -C $(BUILDDIR_VALGRIND) $(VERBOSE) + +$(BUILDDIR_VALGRIND): + @$(MESON) setup --buildtype=debug $(BUILDDIR_VALGRIND) + +vg_opts:=--enable-debuginfod=no --leak-check=full --error-exitcode=1 +test-valgrind: export G_SLICE=always-malloc +test-valgrind: export G_DEBUG=gc-friendly +test-valgrind: build-valgrind + @$(MESON) test -C $(BUILDDIR_VALGRIND) \ + --wrap="$(VALGRIND) $(vg_opts)" \ + --timeout-multiplier 100 + +check-valgrind: test-valgrind + +# we do _not_ pass helgrind; but this seems to be a false-alarm +# https://gitlab.gnome.org/GNOME/glib/-/issues/2662 +test-helgrind: $(BUILDDIR_VALGRIND) + $(MESON) -C $(BUILDDIR_VALGRIND) test \ + --wrap="$(VALGRIND) --tool=helgrind --error-exitcode=1" \ + --timeout-multiplier 100 + +check-helgrind: test-helgrind + +# +# benchmarking +# + +$(BUILDDIR_BENCHMARK): + @$(MESON) setup --buildtype=debugoptimized $(BUILDDIR_BENCHMARK) + +build-benchmark-target: $(BUILDDIR_BENCHMARK) + @$(MESON) compile -C $(BUILDDIR_BENCHMARK) $(VERBOSE) + +benchmark: build-benchmark-target + $(NINJA) -C $(BUILDDIR_BENCHMARK) benchmark + +# +# coverage +# + +$(BUILDDIR_COVERAGE): + $(MESON) setup -Db_coverage=true --buildtype=debug $(BUILDDIR_COVERAGE) + +covfile:=$(BUILDDIR_COVERAGE)/meson-logs/coverage.info + +# generate by hand, meson's built-ins are rather inflexible +coverage: $(BUILDDIR_COVERAGE) + @$(MESON) compile -C $(BUILDDIR_COVERAGE) + @$(MESON) test -C $(BUILDDIR_COVERAGE) $(VERBOSE) + $(LCOV) --capture --directory . --output-file $(covfile) + @$(LCOV) --remove $(covfile) '/usr/*' '*guile*' '*thirdparty*' '*/tests/*' '*mime-object*' --output $(covfile) + @$(LCOV) --remove $(covfile) '*mu/mu/*' --output $(covfile) + @mkdir -p $(BUILDDIR_COVERAGE)/meson-logs/coverage + @$(GENHTML) $(covfile) --output-directory $(BUILDDIR_COVERAGE)/meson-logs/coverage/ + @echo "coverage report at: file://$(BUILDDIR_COVERAGE)/meson-logs/coverage/index.html" + +# +# misc +# + +dist: $(BUILDDIR) + $(MESON) compile -C $(BUILDDIR) $(VERBOSE) + $(MESON) dist -C $(BUILDDIR) $(VERBOSE) + +distclean: clean + +HTMLPATH=${BUILDDIR}/mu4e/mu4e +mu4e-doc-html: + @mkdir -p ${HTMLPATH} && cp mu4e/texinfo-klare.css ${HTMLPATH} + @cd mu4e; makeinfo -v -I ${BUILDDIR} -I ${BUILDDIR}/mu4e --html --css-ref=texinfo-klare.css -o ${HTMLPATH} mu4e.texi diff --git a/NEWS.org b/NEWS.org new file mode 100644 index 0000000..442eb4e --- /dev/null +++ b/NEWS.org @@ -0,0 +1,1582 @@ +#+STARTUP:showall +* NEWS (user visible changes & bigger non-visible ones) + +* 1.12 (released on February 24, 2024) + +** Some highlights + + - Significant speedups in both ~mu~ and ~mu4e~ + - Reworked message composition, closer to its Gnus origins which adds many of its features + - Overhauled the query parser; squashing a number of bugs/limitations, incl. dealing + with CJK messages + - Experimental folding of message threads + - Better and faster indexing of HTML messages + - Experimental search by (human) language wit CLD2 + + For details & more, see below. Note a few minor new features were added + _after_ the initial 1.12.0. + +*** mu + + - new command ~mu move~ to move messages across maildirs and/or change their + flags. See the manpage for all the details. + + - ~mu~ commands ~extract~ ~verify~ and ~view~ can now read the message from + standard input; see their man-pages for details + + - ~mu init~ gained the ~--ignored-address~ option for email-addresses / regexps + that should _not_ be included in the contacts-cache (i.e., for ~mu cfind~ and + Mu4e address completion). See the ~mu-init~ manpage for details. + + It's not unusual for ~noreply~-type e-mail addresses to be the majority in + an e-mail corpus; to get rid of those, with something like + ~--ignored-address=/.*no.*reply.*/~ + + - what used to be the ~mu fields~ command has been merged into ~mu info~; i.e., + ~mu fields~ is now ~mu info fields~. + + - ~mu view~ gained ~--format=html~ which compels it to output the HTML body of + the message rather than the (default) plain-text body. See its updated + manpage for details. + + - when encountering an HTML message part during indexing, previously (i.e., + ~mu 1.10~) we would attempt to process that as-is, with HTML-tags etc.; this + is now improved by employing a custom html->text scraper which extracts + the human-readable text from the html. + + - mu querying and (esp.) showing results has been made significantly faster; + e.g., in one big ~mu find~ query we went from ~47s to only ~7s + + - /experimental/: if you build ~mu~ with [[https://github.com/CLD2Owners/cld2][CLD2]] support (available in many Linux + distros), ~mu~ will try to detect the language of the body of e-mail + messages; you can then search by their ISO-639-1 code, e.g.: + ~$ mu find lang:en~ + + the matching is not perfect, and seems to favor non-English if there's a + mostly English message with some other language mixed in. + + this does require re-indexing the database. + + - set the default database batch-size (using the ~mu init~ command) to 50000 + rather than 250000; the latter was too high for systems with limited + memory. You can of course change that with ~--batch-size=...~ + + - restore expansion for path options such as ~--maildir=~/Maildir~ (to e.g. + ~/home/user/Maildir~) for shells that do not do that, such as Bash. + + - overhauled the query-parser; this is (should be) compatible with the older + one, apart from a number of fixes. There is a new option ~--analyze~ for the + ~mu find~ command, which shows the parsed query in a (hopefully) + human-readable s-expression form; this can be used to debug your queries + (this replaces the older ~--format=mquery|xquery~) + + Furthermore, there now support for "ngram"-based indexing and querying, + which is useful for languages/scripts without explicit word-breaks, such + as Chinese/Japanese/Korean. See the *mu-init* manpages, in particular the + ~--support-ngrams~ option, and why you may (or may not) want to enable that. + + - the build has been made reproducible + +*** mu4e + +**** message composer + + - Overhaul of the message composer; it is now closer to the Gnus/Message + composer functions (e.g. the whole mu4e-specific draft setup is gone); + this reduces code size and offers some new capabilities. + + More of the ~message-~ functionality can be used now in ~mu4e~. + + - Variables ~mu4e-compose-signature~, ~mu4e-compose-cite-function~ are gone + (with aliases in place), use ~message-signature~, ~message-cite-function~ + instead. There's a special ~mu4e-message-cite-nothing~ for the case where + you do not want to cite anything. + + - There's a new function ~mu4e-compose-wide-reply~ (bound to =W=) which does a + wide-reply, a.k.a., 'reply to all'. So ~mu4e-compose-reply-recipients~ is + not needed anymore and has been obsoleted (and doesn't do anything). + ~mu4e-compose-reply-ignore-address~ is no longer supported, use + ~message-prune-recipient-rules~ instead. + + Same for ~mu4e-compose-dont-reply-to-self~; roughly the same effect can be + achieved by setting ~message-dont-reply-to-names~ to + ~#'mu4e-personal-or-alternative-address-p~. This only works for + [[info:(message) Wide Reply][wide-replies]]. + + - Another new function is ~mu4e-compose-supersede~ (not bound to any key by + default), with which you can /supersede/ your own messages; that is, send + the message as a kind-of reply to the same recipients. This only works if + you were the sender. + + - The special mailing list handling is gone; ~mu4e-compose-reply~ and + ~mu4e-compose-wide-reply~ should take care of that. There's also + ~message-reply-to-function~ for ultimate control; see [[info:(message) + Reply][info (message) Reply]] for details. + + - ~mu4e-compose-in-new-frame~ has been generalized (in a backward-compatible + way) to ~mu4e-compose-switch~, which lets you decide whether a message + should be composed in the current window (default), a new window or a new + frame. + + - ~mu4e-compose-context-switch~ is gone; it was a little too fragile. Best + change when creating the message (=mu4e= asks you by default, see + ~mu4e-compose-context-policy~). + + - there's a new hook ~mu4e-compose-post-hook~ which fires when message + composition is complete - either a message has been sent, it is postponed, + canceled etc. (1.12.5). + + - iCalendar support is a work-in-progress with the new editor. One change is + that support is now _automatically_ available. + +**** other + + - New command ~mu4e-search-query~ (bound to =c=) which lets you pick a query + (from bookmark / maildir shortcuts) with completion in main / headers / + view buffers. + + - improved support for dealing with attachments and other MIME-parts in the + message view; they gained completions support with annotations in the + minibuffer + + It is possible to save all attachments at once with =C-c C-a=, except with + Helm, which uses its own mechanism for this. This same has been extended + to the MIME-part actions. + + - experimental: support folding message threads (with =TAB= / =S-TAB=). See the + [[info:mu4e:Folding threads][entry in the Mu4e manual]] for further details. + + - mailing list support was modernized a bit; the format changed (see the + ~mu4e-mailing-lists~ and ~mu4e-user-mailing-lists~ docstrings. There is + ~M-x mu4e-mailing-list-info-refresh~ to update to the new values after + changing them. + + - also, there are now actions ('a' in view/header) to get to online archives + for some (selected) mailing-list archives. + + - ~mu4e-quit~ now takes a prefix argument which, if provided, causes it to + bury the mu main buffer, rather than quitting mu. ~mu4e~ will now just + switch the mu4e buffer if it exists (otherwise it starts ~mu4e~). + + - ~mu4e~ queries are much snappier now, due to the mentioned speed-ups in + querying; ~mu4e~ also adds a new optimization =mu4e-mu-allow-temp-file= + (turned off by default), which speed up things further; e.g., for showing + 500 messages (debug build), we went from 642ms to 247ms, given an + in-memory temp file. + + If and how much this helps, depends on your setup, see the + =mu4e-mu-allow-temp-file= docstring for details on how to determine this. + + - Maildir lists are now generated server-side; so e.g. jumping to the 'jo' + /other/ Maildirs used to be quite slow the first time, but is now very fast. + + ~mu4e-cache-maildir-list~ is obsolete / non-functional now. + + - after retrieving mail (~mu4e-update-mail-and-index~), save the output of the + retrieval command in a buffer =*mu4e-last-update*=, = which can be useful + for diagnosis. + + - links (in text-mode emails) are now clickable through , to be + consistent with eww. + + - support new-mail notifications on MacOS out-of-the-box + + - allow sorting by tag + + - ~mu4e~ now follows Emacs' ~package~ guidelines + +*** Contributors + + Thanks to our contributors - code committers belows, but also to everyone + who filed tickets, asked questions, answered them etc. + + Babak Farrokhi, Christophe Troestler, Christoph Reichenbach, Daniel Fleischer, + David Edmondson, Davide Masserut, Dirk-Jan C. Binnema, Jeremy Sowden, + Lin Jian, Martin R. Albrecht, Nacho Barrientos, Nicholas Vollmer, + Nicolas P. Rougier, ramon diaz-uriarte (at Phelsuma), reindert, Ruijie Yu, + Sean Farley, stardiviner, Tassilo Horn and Thierry Volpiatto + + +* Old news + :PROPERTIES: + :VISIBILITY: folded + :END: + +** 1.10 (released on March 26, 2023) + +*** mu + + - a new command-line parser, which allows (hopefully!) for a better user + interaction; better error checking and more + + - Invalid e-mail addresses are no longer added to the contacts-cache. + + - The ~cfind~ command gained ~--format=json~, which makes it easy to further + process contact information, e.g. using ~jq~. See the manpage for more + details. + + - The ~init~ command learned ~--reinit~ to reinitialize the database with the + settings of an existing one + + - The ~script~ command is gone, and integrated with ~mu~ directly, i.e. the + scripts (when enabled) are directly visible in the ~mu~ output. Also see the + Guile section. + + - The ~extract~ command gained the ~--uncooked~ option to tell it to _not_ replace + spaces with dashes in extracted filenames (and a few other things). + + - Revamped manpages which are now generated from ~org~ descriptions + + - Standardize on PCRE-flavored regular expressions throughout *mu*. + + - ~mu~ no longer attempts to 'expand' the =~= (and some other characters) in + command line options that take filenames, since it was a bit unpredictable. + So write e.g. ~--option=/home/user/hello~ instead of ~--option=~/hello~ + + - Experimental: as bit of a hack, html message bodies are processed as if + they were plain text, similar how "old mu" would do it (1.6.x and earlier). + A nicer solution would be to convert to text, but this something for the + future. + + - the MSYS2 (Windows) builds is _experimental_ now; some things may not work; + see e.g. https://github.com/djcb/mu/issues?q=is%3Aissue+label%3Amsys, but + we welcome efforts to fix those things. + +*** mu4e + + - ~emacs~ 26.3 or higher is now required for ~mu4e~ + + - ~mu4e-view-mode-hook~ now fires before the message is rendered. If you have + hook-functions that depend on the message contents, you should use + the new ~mu4e-view-rendered-hook~. + + - mu4e window management has been completely reworked and cleaned up, + affecting the message loading as well as the window-layout. As a + user-visible feature, there's now the =z= binding (~mu4e-view-detach~), to + 'detach' view and alllow for keV Detaching and reattaching][manual entry]] for further + details. + + - As a result of that, ~mu4e-split-view~ can no longer be a function; the new + way is to use ~display-buffer-alist~ as explained in the [[info:mu4e:Buffer Display][manual]] + + - ~mu4e~ now keeps track of 'baseline' query results and shows the difference + from that in the main view and modeline (you'll might see something like + =1(+1)/2= for your bookmarks or in the modeline; that means that there is + one more unread message since baseline; see the [[info:mu4e#Bookmarks and Maildirs][manual entry]] for details. + + The idea is that you get a quick overview of where changes happened while + you were doing something else. This is a somewhat experimental feature + which is under active development + + - Related to that, you can now crown one of your bookmarks in =mu4e-bookmarks= + with ~:favorite t~, causing it to be highlighted in the main view and used + in the mode-line. See the new [[info:mu4e#Modeline][modeline entry]] in the manual; this uses the + new =mu4e-modeline-mode= minor-mode. + + - Expanding on that further, you can also get desktop notifications for new + mail (on systems with DBus for now; see [[info:mu4e:#Desktop notifications][Desktop notifications]] in the + manual. + + - If your search query matches some bookmark, the modeline now shows the + bookmark's name rather than the query; this can be controlled through + =mu4e-modeline-prefer-bookmark-name= (default: =t=). + + - You can now tell mu4e to use emacs' completion system rather than the mu4e + built-in one; see the variables ~mu4e-read-option-use-builtin~ and + ~mu4e-completing-read-function~; e.g. to always emacs completion (which + may have been enhanced by various completion frameworks), use: + #+begin_src elisp + (setq mu4e-read-option-use-builtin nil + mu4e-completing-read-function 'completing-read) + #+end_src + + - when moving messages (which includes changing flags), file-flags changes + are propagated to duplicates of the messages; that is, e.g. the /Seen/ or + /Replied/ status is propagated to all duplicates (earlier, this was only + done when marking a message as read). Note, /Draft/, /Flagged/ and /Trashed/ + flags are deliberately *not* propagated. + + - Teach ~mu4e-copy-thing-at-point~ about ~shr~ links + + - The ~mu4e-headers-toggle-setting~ has been renamed + ~mu4e-headers-toggle-property~ and has the new default binding ~P~, which + works in both the headers-view and message-view. The older functions + ~mu4e-headers-toggle-threading~, ~mu4e-headers-toggle-threading~, + ~mu4e-headers-toggle-full-search~ ~mu4e-headers-toggle-include-related~, + ~full-search~skip-duplicates~ have been removed (with their keybindings) in + favor of ~mu4e-headers-toggle-property~. + + - There's also a new property ~mu4e-headers-hide-enabled~, which controls + wheter ~mu4e-headers-hide-predicate~ is applied (when non-~nil~). This can be + used to temporarily turn the predicate off/on. + + - You can now jump to previous / next threads in headers-view, message view. + Default binding is ~{~ and ~}~, respectively. + + - When searching, the number of hidden messages is now shown in the + message footer along with the number of Found messages + + - The ~eldoc~ support in header-mode is now optional and disabled by default; + set ~mu4e-eldoc-support~ to non-nil to enable it. + + - In the main view, the keybindings shown are a representation of the actual + keybindings, rather than just the defaults. This is for the benefit for + people who want to use different keybindings. + + - As a side-effect of that, ~mu4e-main-mode~ and ~mu4e-main-mode-hook~ functions + are now invoked _before_ the rendering takes place; if you're customizations + depend on happening after rendering is completed, use the new + ~mu4e-main-rendered-hook~ instead. + + - ~mu4e-cache-maildir-list~ has been promoted to be a =defcustom=, enabled by + default. This caches the list of "other" maildirs (i.e., without a + shortcut). + + - For testing, a new command ~mu4e-server-repl~ to start a ~mu~ server just as + ~mu4e~ does it. Note that this cannot run at the same time when ~mu4e~ runs. + + - all the obsolete function and variable aliases have been moved to + ~mu4e-obsolete.el~ so we can unclutter the non-obsolete code a bit. + +*** guile + + - in the 1.8 release, the /current/ Guile API was deprecated; that does not + mean that Guile support goes way, just that it will look different. + + - Guile script commands are now integrated with the main ~mu~, so without + further parameters ~mu~ shows both subcommands and scripts. This is a + work-in-progress! + + - The per-(week|day|year|year-month) scripts have been combined into a + ~histogram~ script. If you have Guile-support enabled, and have ~gnuplot~ + installed, you can do e.g., + +#+begin_example + mu histogram -- --time-unit=day --query="hello" +#+end_example + + to get a histogram of such messages. Note, this area is under active + development and will likely change. + +*** building and installation + + - the autotools build (which was deprecated since 1.8) has now been removed. + we thank it for its services since 2008. We continue with ~meson~. + + However, we still have ~autogen.sh~ and a ~Makefile~ which can be helpful for + driving ~meson~-based builds. Think of the ~Makefile~ as a convenient place to + put common action for which I always forget the ~meson~ incantation.** + + - ~meson~ 56.0 or higher is required for building + + - ~emacs~ 26.3 or higher is needed for ~mu4e~ + +*** internals + + As usual, there have been a number of internal updates in the ~mu~ codebase: + + - reworked the internal s-expression parser + + - new command-line argument parser (based on CLI11) + + - message-move flag propagation moved from the mu4e-server to mu-store + + - more =mu4e~= internals have been renamed/reworked in to ~mu4e--~. + +*** contributor to this release + + Aimé Bertrand, Aleksei Atavin, Al Haji-Ali, Andreas Hindborg, Anton Tetov, + Arsen Arsenović, Babak Farrokhi, Ben Cohen, Damon Kwok, Daniel Colascione, + Derek Zhou, Dirk-Jan C. Binnema, John Hamelink, Leo Gaskin, Manuel + Wiesinger, Marcel van der Boom, Mark Knoop, Mickey Petersen, Nicholas + Vollmer, Protesilaos Stavrou, Remco van 't Veer, Sean Allred, Sean Farley, + Stephen Eglen, Tassilo Horn + + And of course all the people how filed tickets, asked question, provided + suggestions. + + +** 1.8 (released on June 25, 2022) + + (there are some changes in the installation procedure compared to 1.6.x; see + Installation below) + +**** mu + + - The server protocol (as used my mu4e) has seen a number of updates, to + allow for faster rendering. As before, there's no compatibility between + minor release numbers (1.4 vs 1.6 vs 1.8) nor within development series + (such as 1.7). However, within a stable release (such as all 1.6.x) the + protocol won't change (except if required to fix some severe bug; this + never happened in practice) + + - The ~processed~ number in the indexing statistics has been renamed into + ~checked~ and describes the number of message files considered for updating, + which is a bit more useful that the old value, which was more-or-less + synonymous with the ~updated~ number (which are the messages that got + (re)parsed / (re)added to the database. + + Basically, it counts all the messages for which we checked their timestamp. + + - The internals of the message handling in ~mu~ have been heavily reworked; + much of this is not immediately visible but is an enabler for some new + features. + + - instead of passing ~--muhome~, you can now also set an environment variable + ~MUHOME~. + + - the ~info~ command now includes information about the last indexing + operation and the last database change that took place; note that the + information may be slightly delayed due to database caching. + + - the ~verify~ command for checking signatures has been updated, and is more + informative + + - a new command ~fields~ provides information about the message fields and + flags for use in queries. The information is the same information that ~mu~ + uses and so stays up to date. + + - a new message field ~changed~, which refers to the time/date of the last + time a message was changed (the file ~ctime~) + + - new message flags ~personal~ to search for "personal" messages, which are + defined as a message with at least one personal contact, and ~calendar~ for + messages with calendar-invitations. + + - message sexps are now cached in the store, which makes delivering + sexp-based search results (as used by ~mu4e~) much faster. + + - Windows/MSYS support is deprecated; it doesn't work well (if at all) and + there's currently not sufficient developer interest/expertise to change + this. + +**** mu4e + + - the old mu4e-view is *gone*; only the gnus-based one remains. This allowed + for removing quite a bit of old code. + + - the mu4e headers rendering is much faster (a factor of 3+), which makes + displaying big results snappier. This required some updates in the headers + handling and in the server protocol. Separate from that, the cached + message sexps (see the ~mu~ section) make getting the results much faster. + This becomes esp. clear when there are a lot of query results. + + - "related" messages are now recognizable as such in the headers-view, with + their own face, ~mu4e-related-face~; by default with an italic slant. + + - For performance testing, you can set the variable + ~mu4e-headers-report-render-time~ to ~t~ and ~mu4e~ will report the + search/rendering speed of each query operation. + + - Removed header-fields ~:attachments~, ~:signature~, ~:encryption~ and + ~:user-agent~. They're obsolete with the Gnus-based message viewer. + + - The various "toggles" for the headers-view (full-search, include-related, + skip-duplicates, threading) were a bit hard to find and with non-obvious + key-bindings. For that, there is now ~mu4e-headers-toggle-setting~ (bound + to ~M~) to handle all of that. The toggles are also reflected in the + mode-line; so e.g. 'RTU' means we're including [R]elated messages, and show + [T]hreads, skip duplicates ([U]nique). + + - A new ~defcustom~, ~mu4e-view-open-program~ for starting the appropriate + program for a give file (e.g., ~xdg-open~). There are some reasonable + defaults for various systems. This can also be set to a function. + + - indexing happens in the background now and mu4e can interact with the + server while it is ongoing; this allows for using mu4e during lengthy + indexing operations. + + - ~mu4e-index-updated-hook~ now fires after indexing completed, regardless of + whether anything changed (before, it fired only if something changed). In + your hook-functions (or elsewhere) you can check if anything changed using + the new variable ~mu4e-index-update-status~. And note that ~processed~ has + been renamed into ~checked~, with a slightly different meaning, see the mu + section. + + - ~message-user-organization~ can now be used to set the ~Organization:~ + header. See its docstring for details. + + - ~mu4e-compose-context-switch~ no longer attempts to update the draft folder + (which turned out to be a little fragile). However, it has been updated to + automatically change the ~Organization:~ header, and attempts to update the + message signature. Also, there's a key-binding now: ~C-c ;~ + + - Changed the default for ~mu4e-compose-complete-only-after~ to 2018-01-01, + to filter out contacts not seen after that date. + + - As an additional measure to limit the number of contacts that mu4e loads + for auto-completions, there's ~mu4e-compose-complete-max~, to set a precise + numerical match (*before* any possible filtering). Set to ~nil~ (no maximum + by default). + + - Updated the "fancy" characters for some header fields. Added new ones for + personal and list messages. + + - Removed ~make-mu4e-bookmark~ which was obsoleted in version 1.3.9. + + - Add command ~mu4e-sexp-at-point~ for showing/hiding the s-expression for + the message-at-point. Useful for development / debugging. Bound to ~,~ in + headers and view mode. + + - undo is now supported across message-saves + + - a lot of the internals have been changed: + + - =mu4e= is slowly moving from using the '=~'= to the more common '=--'= + separator for private functions; i.e., =mu4e-foo= becomes =mu4e--foo=. + + - =mu4e-utils.el= had become a bit of a dumping ground for bits of code; + it's gone now, with the functionality move to topic-specific files -- + =mu4e-folders.el=, =mu4e-bookmarks.el=, =mu4e-update.el=, and included in + existing files. + + - the remaining common functionality has ended up in =mu4e-helpers.el= + + - =mu4e-search.el= takes the search-specific code from =mu4e-headers.el=, + and adds a minor-mode for the keybindings. + + - =mu4e-context.el= and =mu4e-update.el= also define minor modes with + keybindings, which saves a lot of code in the various views, since they + don't need explicitly bind all those function. + + - also =mu4e-vars.el= had become very big, we're refactoring the =defvar= / + =defcustom= declarations to the topic-specific files. + + - =mu4e-proc.el= has been renamed =mu4e-server.el=. + + - Between =mu= and =mu4e=, contact cells are now represented as a plist ~(:name + "Foo Bar" :email "foobar@example.com")~ rather than a cons-cell ~("Foo + Bar" . "foobar@example.com").~ + + If you have scripts depending on the old format, there's the + ~mu4e-contact-cons~ function which takes a news-style contact and yields + the old form. + + - Because of all these changes, it is recommended you remove older version + of ~mu4e~ before reinstalling. + +**** guile + + - the current guile support has been deprecated. It may be revamped at some + point, but will be different from the current one, which is to be removed + after 1.8 + +**** toys + + - the ~toys~ (~mug~) has been removed, as they no longer worked with the rest of + the code. + +*** Installation + + - =mu= switched to the [[https://mesonbuild.com][meson]] build system by default. The existing =autotools= + is still available, but is to be removed after the 1.8 release. + + Using =meson= (which you may need to install), you can use something like + the following in the mu top source directory: + +#+BEGIN_SRC sh + $ meson build && ninja -C build +#+END_SRC + + - However, note that =autogen.sh= has been updated, and there's a + convenience =Makefile= with some useful targets, so you can also do: +#+BEGIN_SRC sh + $ ./autogen.sh && make # and optionally, 'sudo make install' +#+END_SRC + + - After that, either =ninja -C build= or =make= should be enough to rebuild + + - NOTE: development versions 1.7.18 - 17.7.25 had a bug where the mail file + names sometimes got misnamed (with some extra ':2,'). This can be restored + with something like: +#+begin_example + $ find ~/Maildir -name '*:2,*:*' | \ + sed "s/\(\([^:]*\)\(:2,\)\{1,\}\(:2,.*$\)\)/mv '\0' '\2\4'/" > rename.sh +#+end_example + (replace 'Maildir' with the path to your maildir) + + once this is done, do check the generated 'rename.sh' and after convincing + yourself it does the right thing, do +#+begin_example + $ sh rename.sh +#+end_example + after that, re-index. + + - Before installing, it is recommended that you *remove* any older versions + of ~mu~ and especially ~mu4e~, since they may conflict with the newer ones. + + - =mu= now requires C++17 support for building + + +*** Contributor for this release + + - As per ~git~: c0dev0id, Christophe Troestler, Daniel Fleischer, Daniel Nagy, + Dirk-Jan C. Binnema, Dr. Rich Cordero, Kai von Fintel, Marcelo Henrique + Cerri, Nicholas Vollmer, PRESFIL, Tassilo Horn, Thierry Volpiatto, Yaman + Qalieh, Yuri D'Elia, Zero King + - And of course all the people filing issues, suggesting features and helping + out on the maling list. + + + + +** 1.6 (released, as of July 27 2021) + + NOTE: After upgrading, you need to call ~mu init~, with your prefered parameters + before you can use ~mu~ / ~mu4e~. This is because the underlying database-schema + has changed. + +*** mu + + - Where available (and with suitably equiped ~libglib~), log to the ~systemd~ + journal instead of =~/.cache/mu.log=. Passing the ~--debug~ option to ~mu~ + increases the amount that is logged. + + - Follow symlinks in maildirs, and support moving messsages across + filesystems. Obviously, that is typically quite a bit slower than the + single-filesystem case, but can be still be useful. + + - Optionally provide readline support for the ~mu~ server (when in tty-mode) + + - Reworked the way mu generates s-expressions for mu4e; they are created + programmatically now instead of through string building. + + - The indexer (the part of mu that scans maildirs and updates the message + store) has been rewritten so it can work asynchronously and take advantage + of multiple cores. Note that for now, indexing in ~mu4e~ is still a blocking + operation. + + - Portability updates for dealing with non-POSIX systems, and in particular + VFAT filesystem, and building using Clang/libc++. + + - The personal addresses (as per ~--my-address=~ for ~mu init~) can now also + include regular expressions (basic POSIX); wrap the expression in ~/~, e.g., + ~--my-address='/.*@example.*/~'. + + - Modernized the querying/threading machinery; this makes some old code a + lot easier to understand and maintain, and even while not an explicit + goal, is also faster. + + - Experimental support for the Meson build system. + +*** mu4e + + - Use the gnus-based message viewer as the default; the new viewer has quite + a few extra features compared to the old, mu4e-specific one, such as + faster crypto, support for S/MIME, syntax-highlighting, calendar + invitations and more. + + The new view is superior in most ways, but if you still depend on + something from the old one, you can use: + #+begin_example + ;; set *before* loading mu4e; and restart emacs if you want to change it + ;; users of use-packag~ should can use the :init section for this. + (setq mu4e-view-use-old t) + #+end_example + + (The older variable ~mu4e-view-use-gnus~ with the opposite meaning is + obsolete now, and no longer in use). + + - Include maildir-shortcuts in the main-view with overall/unread counts, + similar to bookmarks, and with the same ~:hide~ and ~:hide-unread~ properties. + Note that for the latter, you need to update your maildir-shortcuts to the + new format, as explained in the ~mu4e-maildir-shortcuts~ docstring. + + You can set ~mu4e-main-hide-fully-read~ to hide any bookmarks/maildirs that + have no unread messages. + + - Add some more properties for use in capturing org-mode links to messages / + queries. See [[info:mu4e#Org-mode links][the mu4e manual]] for details. + + - Honor ~truncate-string-ellipsis~ so you can now use 'fancy' ellipses for + truncated strings with ~(setq truncate-string-ellipsis "…")~ + + - Add a variable ~mu4e-mu-debug~ which, when set to non-~nil,~ makes the ~mu~ + server log more verbosely (to ~mu.log~ or the journal) + + - Better alignment in headers-buffers; this looks nicer, but is also a bit + slower, hence you need to enable ~mu4e-headers-precise-alignment~ for this. + + - Support ~mu~'s new regexp-based personal addresses, and add + ~mu4e-personal-address-p~ to check whether a given string matches a personal + address. + + - TAB-Completion for writing ~mu~ queries + + - Switch the context for existing draft messages using + ~mu4e-compose-context-switch~ or ~C-c C-;~ in ~mu4e-compose-mode~. + + +** 1.4 (released, as of April 18 2020) + +*** mu + + - mu now defaults to the [[https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html][XDG Base Directory Specification]] for the default + locations for various files. E.g. on Unix the mu database now lives under + ~~/.cache/mu/~ rather than ~~/.mu~. You can still use the old location by + passing ~--muhome=~/.mu~ to various ~mu~ commands, or setting ~(setq + mu4e-mu-home "~/.mu")~ for ~mu4e~. + + If your ~~/.cache~ is volatile (e.g., is cleared on reboot), you may want + use ~--muhome~. Some mailing-list dicussion suggest that's fairly rare + though. + + After upgrading, you may wish to delete the files in the old location to + recover some diskspace. + + - There's a new subcommand ~mu init~ to initialize the mu database, which + takes the ~--maildir~ and ~--my-address~ parameters that ~index~ used to take. + These parameters are persistent so ~index~ does not need (or accept) them + anymore. ~mu4e~ now depends on those parameters. + + ~init~ only needs to be run once or when changing these parameters. That + implies that you need to re-index after changing these parameters. The + ~.noupdate~ files are ignored when indexing the first time after ~mu init~ (or + in general, when the database is empty). + + - There is another new subcommand ~mu info~ to get information about the mu + database, the personal addresses etc. + + - The contacts cache (which is used by ~mu cfind~ and ~mu4e~'s + contact-completion) is now stored as part of the Xapian database rather + than as a separate file. + + - The ~--xbatchsize~ and ~--autoupgrade~ options for indexing are gone; both are + determined implicitly now. + +*** mu4e + + - ~mu4e~ no longer uses the ~mu4e-maildir~ and ~mu4e-user-mail-address-list~ + variables; instead it uses the information it gets from ~mu~ (see the ~mu~ + section above). If you have a non-default ~mu4e-mu-home~, make sure to set + it before ~mu4e~ starts. + + It is strongly recommended that you run ~mu init~ with the appropriate + parameters to (re)initialize the Xapian database, as mentioned in the + mu-section above. + + The main screen shows your address(es), and issues a warning if + ~user-email-address~ is not part of that (and refer you to ~mu init~). You can + avoid the addresses in the main screen and the warning by setting + ~mu4e-main-view-hide-addresses~ to non-nil. + + - In many cases, ~mu4e~ used to receive /all/ contacts after each indexing + operation; this was slow for some users, so we have updated this to /only/ + get the contacts that have changed since the last round. + + We also moved sorting the contacts to the mu-side, which speeds things up + further. However, as a side-effect of this, ~mu4e-contact-rewrite-function~ + and ~mu4e-compose-complete-ignore-address-regexp~ have been obsoleted; users + of those should migrate to ~mu4e-contact-process-function~; see its + docstring for details. + + - Christophe Troestler contributed support for Gnus' calender-invitation + handling in mu4e (i.e., you should be able to accept/reject invitations + etc.). It's very fresh code, and likely it'll be tweaked in the future. + But it's available now for testing. Note that this requires the gnus-based + viewer, as per ~(setq mu4e-view-use-gnus t)~ + + - In addition, he added support for custom headers, so the ones for for the + non-gnus-view should work just as well. + + - ~org-mode~ support is enabled by default now. ~speedbar~ support is disabled + by default. The support org functionality has been moved to ~mu4e-org.el~, + with ~org-mu4e.el~ remaining for older things. + + - ~mu4e~ now adds message-ids to messages when saving drafts, so we can find + them even with ~mu4e-headers-skip-duplicates~. + + - Bookmarks (as in ~mu4e-bookmarks~) are now simple plists (instead of cl + structs). ~make-mu4e-bookmark~ has been updated to produce such plists (for + backward compatibility). A bookmark now looks like a list of e.g. ~(:name + "My bookmark" :query "banana OR pear" :key ?f)~ this format is a bit easier + extensible. + + - ~mu4e~ recognizes an attribute ~:hide t~, which will hide the bookmark item + from the main-screen (and speedbar), but keep it available through the + completion UI. + + - ~mu4e-maildir-shortcuts~ have also become plists. The older format is still + recognized for backward compatibility, but you are encouraged to upgrade. + + - Replying to mailing-lists has been improved, allowing for choosing for + replying to all, sender, list-only. + + - A very visible change, ~mu4e~ now shows unread/all counts for bookmarks in + the main screen that are strings. This is on by default, but can be + disabled by setting ~:hide-unread~ in the bookmark ~plist~ to ~t~. For + speed-reasons, these counts do _not_ filter out duplicates nor messages that + have been removed from the filesystem. + + - ~mu4e-attachment-dir~ now also applies to composing messages; it determines + the default directory for inclusion. + + - The mu4e <-> mu interaction has been rewritten to communicate using + s-expressions, with a repl for testing. + +*** guile + + - guile 3.0 is now supported; guile 2.2 still works. + +*** toys + + - Updated the ~mug~ toy UI to use Webkit2/GTK+. Note that this is just a toy + which is not meant for distribution. ~msg2pdf~ is disabled for now. + + +*** How to upgrade mu4e + + - upgrade ~mu~ to the latest stable version (1.4.x) + + - shut down emacs + + - Run ~mu init~ in a terminal + + - Make sure ~mu init~ points to the right Maildir folder and add your email + address(es) the following way: + + ~mu init --maildir=~/Maildir --my-address=jim@example.com --my-address=bob@example.com~ + + - once this is done, run ~mu index~ + + - Don't forget to delete your old mail cache location if necessary (see + release notes for more detail). + +** 1.2 + + After a bit over a year since version 1.0, here is version 1.2. This is + mostly a bugfix release, but there are also a number of new features. + +*** mu + + - Substantial (algorithmic) speed-up of message-threading; this also (or + especially) affects mu4e, since threading is the default. See commit + eb9bfbb1ca3c for all the details, and thanks to Nicolas Avrutin. + + - The query-parser now generates better queries for wildcard searches, by + using the Xapian machinery for that (when available) rather than + transforming into regexp queries. + + - The perl backend is hardly used and will be removed; for now we just + disable it in the build. + + - Allow outputting messages in json format, closely following the sexp + output. This adds an (optional) dependency on the Json-Glib library. + +*** mu4e + + - Bump the minimal required emacs version to 24.4. This was already de-facto + true, now it is enforced. + + - In mu4e-bookmarks, allow the `:query` element to take a function (or + lambda) to dynamically generate the query string. + + - There is a new message-view for mu4e, based on the Gnus' article-view. + This bring a lot of (but not all) of the very rich Gnus article-mode + feature-set to mu4e, such as S/MIME-support, syntax-highlighting, + + For now this is experimental ("tech preview"), but might replace the + current message-view in a future release. Enable it with: + (setq mu4e-view-use-gnus t) + + Thanks to Christophe Troestler for his work on fixing various encoding + issues. + + - Many bug fixes + +*** guile + + - Now requires guile 2.2. + +*** Contributors for this release: + + Ævar Arnfjörð Bjarmason, Albert Krewinkel, Alberto Luaces, Alex Bennée, Alex + Branham, Alex Murray, Cheong Yiu Fung, Chris Nixon, Christian Egli, + Christophe Troestler, Dirk-Jan C. Binnema, Eric Danan, Evan Klitzke, Ian + Kelling, ibizaman, James P. Ascher, John Whitbeck, Junyeong Jeong, Kevin + Foley, Marcelo Henrique Cerri, Nicolas Avrutin, Oleh Krehel, Peter W. V. + Tran-Jørgensen, Piotr Oleskiewicz, Sebastian Miele, Ulrich Ölmann, + +** 1.0 + + After a decade of development, *mu 1.0*! + + Note: the new release requires a C++14 capable compiler. + +*** mu + + - New, custom query parser which replaces Xapian's 'QueryParser' + both in mu and mu4e. Existing queries should still work, but the new + engine handles non-alphanumeric queries much better. + - Support regular expressions in queries (with the new query engine), + e.g. "subject:/foo.*bar/". See the new `mu-query` and updated `mu-easy` + manpages for examples. + - cfind: ensure nicks are unique + - auxiliary programs invoked from mu/mu4e survive terminating the + shell / emacs + +*** mu4e + + - Allow for rewriting message bodies + - Toggle-menus for header settings + - electric-quote-(local-)mode work when composing emails + - Respect format=flowed and delsp=yes for viewing plain-text + messages + - Added new mu4e-split-view mode: single-window + - Add menu item for `untrash'. + - Unbreak abbrevs in mu4e-compose-mode + - Allow forwarding messages as attachments + (`mu4e-compose-forward-as-attachment') + - New defaults: default to 'skip duplicates' and 'include related' + in headers-view, which should be good defaults for most users. Can be + customized using `mu4e-headers-skip-duplicates' and + `mu4e-headers-include-related', respectively. + - Many bug fixed (see github for all the details). + - Updated documentation + +*** Contributors for this release: + + Ævar Arnfjörð Bjarmason, Alex Bennée, Arne Köhn, Christophe Troestler, + Damien Garaud, Dirk-Jan C. Binnema, galaunay, Hong Xu, Ian Kelling, John + Whitbeck, Josiah Schwab, Jun Hao, Krzysztof Jurewicz, maxime, Mekeor Melire, + Nathaniel Nicandro, Ronald Evers, Sean 'Shaleh' Perry, Sébastien Le + Callonnec, Stig Brautaset, Thierry Volpiatto, Titus von der Malsburg, + Vladimir Sedach, Wataru Ashihara, Yuri D'Elia. + + And all the people on the mailing-list and in github, with bug reports, + questions and suggestions. + + +** 0.9.18 + + New development series which will lead to 0.9.18. + +*** mu + + - Increase the default maximum size for messages to index to 500 + Mb; you can customize this using the --max-msg-size parameter to mu index. + - implement "lazy-checking", which makes mu not descend into + subdirectories when the directory-timestamp is up to date; greatly speeds + up indexing (see --lazy-check) + - prefer gpg2 for crypto + - fix a crash when running on OpenBSD + - fix --clear-links (broken filenames) + - You can now set the MU_HOME environment variable as an + alternative way of setting the mu homedir via the --muhome command-line + parameter. + +*** mu4e + +**** reading messages + + - Add `mu4e-action-view-with-xwidget`, and action for viewing + e-mails inside a Webkit-widget inside emacs (requires emacs 25.x with + xwidget/webkit/gtk3 support) + - Explicitly specify utf8 for external html viewing, so browsers + can handle it correctly. + - Make `shr' the default renderer for rich-text emails (when + available) + - Add a :user-agent field to the message-sexp (in mu4e-view), which + is either the User-Agent or X-Mailer field, when present. + +**** composing messages + + - Cleanly handle early exits from message composition as well as while + composing. + - Allow for resending existing messages, possibly editing them. M-x + mu4e-compose-resend, or use the menu; no shortcut. + - Better handle the closing of separate compose frames + - Improved font-locking for the compose buffers, and more extensive + checks for cited parts. + - automatically sign/encrypt replies to signed/encrypted messages + (subject to `mu4e-compose-crypto-reply-policy') + +**** searching & marking + + - Add a hook `mu4e-mark-execute-pre-hook`, which is run just before + executing marks. + - Just before executing any search, a hook-function + `mu4e-headers-search-hook` is invoked, which receives the search + expression as its parameter. + - In addition, there's a `mu4e-headers-search-bookmark-hook` which + gets called when searches get invoked as a bookmark (note that + `mu4e-headers-search-hook` will also be called just afterwards). This + hook also receives the search expression as its parameter. + - Remove the 'z' keybinding for leaving the headers + view. Keybindings are precious! + - Fix parentheses/precedence in narrowing search terms + +**** indexing + + - Allow for indexing in the background; see + `mu4e-index-update-in-background`. + - Better handle mbsync output in the update buffer + - Add variables mu4e-index-cleanup and mu4e-index-lazy to enable + lazy checking from mu4e; you can sit from mu4e using something like: +#+begin_src elisp +(setq mu4e-index-cleanup nil ;; don't do a full cleanup check + mu4e-index-lazy-check t) ;; don't consider up-to-date dirs #+END_SRC +#+end_src +**** misc + + - don't overwrite global-mode-string, append to it. + - Make org-links (and more general, all users of + mu4e-view-message-with-message-id) use a headers buffer, then view the + message. This way, those linked message are just like any other, and can + be deleted, moved etc. + - Support org-mode 9.x + - Improve file-name escaping, and make it support non-ascii filenames + - Attempt to jump to the same messages after a re-search update operation + - Add action for spam-filter options + - Let `mu4e~read-char-choice' become case-insensitive if there is + no exact match; small convenience that affects most the single-char + option-reading in mu4e. + +*** Perl + + - an experimental Perl binding ("mup") is available now. See + perl/README.md for details. + +*** Contributors: + + Aaron LI, Abdo Roig-Maranges, Ævar Arnfjörð Bjarmason, Alex Bennée, Allen, + Anders Johansson, Antoine Levitt, Arthur Lee, attila, Charles-H. Schulz, + Christophe Troestler, Chunyang Xu, Dirk-Jan C. Binnema, Jakub Sitnicki, + Josiah Schwab, jsrjenkins, Jun Hao, Klaus Holst, Lukas Fürmetz, Magnus + Therning, Maximilian Matthe, Nicolas Richard, Piotr Trojanek, Prashant + Sachdeva, Remco van 't Veer, Stephen Eglen, Stig Brautaset, Thierry + Volpiatto, Thomas Moulia, Titus von der Malsburg, Yuri D'Elia, Vladimir + Sedach + +** 0.9.16 + +*** Release + + 2016-01-20: Release from the 0.9.15 series + +*** Contributors: + + Adam Sampson, Ævar Arnfjörð Bjarmason, Bar Shirtcliff, Charles-H. Schulz, + Clément Pit--Claudel, Damien Cassou, Declan Qian, Dima Kogan, Dirk-Jan C. + Binnema, Foivos S. Zakkak, Hinrik Örn Sigurðsson, Jeroen Tiebout, JJ Asghar, + Jonas Bernoulli, Jun Hao, Martin Yrjölä, Maximilian Matthé, Piotr Trojanek, + prsarv, Thierry Volpiatto, Titus von der Malsburg + + (and of course all people who reported issues, provided suggestions etc.) + +** 0.9.15 + + - bump version to 0.9.15. From now on, odd minor version numbers + are for development versions; thus, 0.9.16 is to be the next stable + release. + - special case text/calendar attachments to get .vcs + extensions. This makes it easier to process those with external tools. + - change the message file names to better conform to the maildir + spec; this was confusing some tools. + - fix navigation when not running in split-view mode + - add `mu4e-view-body-face', so the body-face for message in the + view can be customized; e.g. (set-face-attribute 'mu4e-view-body-face nil + :font "Liberation Serif-10") + - add `mu4e-action-show-thread`, an action for the headers and view + buffers to search for messages in the same thread as the current one. + - allow for transforming mailing-list names for display, using + `mu4e-mailing-list-patterns'. + - some optimizations in indexing (~30% faster in some cases) + - new variable mu4e-user-agent-string, to customize the User-Agent: + header. + - when removing the "In-reply-to" header from replies, mu4e will + also remove the (hidden) References header, effectively creating a new + message-thread. + - implement 'mu4e-context', for defining and switching between + various contexts, which are groups of settings. This can be used for + instance for switch between e-mail accounts. See the section in the manual + for details. + - correctly decode mailing-list headers + - allow for "fancy" mark-characters; and improve the default set + - by default, the maildirs are no longer cached; please see the + variable ~mu4e-cache-maildir-list~ if you have a lot of maildirs and it + gets slow. + - change the default value for + ~org-mu4e-link-query-in-headers-mode~ to ~nil~, ie. by default link to the + message, not the query, as this is usually more useful behavior. + - overwrite target message files that already exist, rather than + erroring out. + - set mu4e-view-html-plaintext-ratio-heuristic to 5, as 10 was too + high to detect some effectively html-only messages + - add mu4e-view-toggle-html (keybinding: 'h') to toggle between + text and html display. The existing 'mu4e-view-toggle-hide-cited' gets the + new binding '#'. + - add a customization variable `mu4e-view-auto-mark-as-read' + (defaults to t); if set to nil, mu4e won't mark messages as read when you + open them. This can be useful on read-only file-systems, since + marking-as-read implies a file-move operation. + - use smaller chunks for mu server on Cygwin, allowing for better + mu4e support there. + +** 0.9.13 + +*** contributors + + Attila, Daniele Pizzolli, Charles-H.Schulz, David C Sterrat, Dirk-Jan C. + Binnema, Eike Kettner, Florian Lindner, Foivos S. Zakkak, Gour, KOMURA + Takaaki, Pan Jie, Phil Hagelberg, thdox, Tiago Saboga, Titus von der + Malsburg + + (and of course all people who reported issues, provided suggestions etc.) + +*** mu/mu4e/guile + + - NEWS (this file) is now visible from within mu4e – "N" in the main-menu. + + - make `mu4e-headers-sort-field', `mu4e-headers-sort-direction' + public (that, is change the prefix from mu4e~ to mu4e-), so users can + manipulate them + + - make it possible the 'fancy' (unicode) characters separately for + headers and marks (see the variable `mu4e-use-fancy-chars'.) + + - allow for composing in a separate frame (see + `mu4e-compose-in-new-frame') + + - add the `:thread-subject' header field, for showing the subject + for a thread only once. So, instead of (from the manual): + +#+begin_example +06:32 Nu To Edmund Dantès GstDev + Re: Gstreamer-V4L... +15:08 Nu Abbé Busoni GstDev + Re: Gstreamer-V... +18:20 Nu Pierre Morrel GstDev \ Re: Gstreamer... +2013-03-18 S Jacopo EmacsUsr + emacs server on win... +2013-03-18 S Mercédès EmacsUsr \ RE: emacs server ... +2013-03-18 S Beachamp EmacsUsr + Re: Copying a whole... +22:07 Nu Albert de Moncerf EmacsUsr \ Re: Copying a who... +2013-03-18 S Gaspard Caderousse GstDev | Issue with GESSimpl... +2013-03-18 Ss Baron Danglars GuileUsr | Guile-SDL 0.4.2 ava... +End of search results +#+end_example + +the headers list would now look something like: +#+begin_example +06:32 Nu To Edmund Dantès GstDev + Re: Gstreamer-V4L... +15:08 Nu Abbé Busoni GstDev + +18:20 Nu Pierre Morrel GstDev \ Re: Gstreamer... +2013-03-18 S Jacopo EmacsUsr + emacs server on win... +2013-03-18 S Mercédès EmacsUsr \ +2013-03-18 S Beachamp EmacsUsr + Re: Copying a whole... +22:07 Nu Albert de Moncerf EmacsUsr \ +2013-03-18 S Gaspard Caderousse GstDev | Issue with GESSimpl... +2013-03-18 Ss Baron Danglars GuileUsr | Guile-SDL 0.4.2 ava... +End of search results +#+end_example + + This is a feature known from e.g. `mutt' and `gnus` and many other + clients, and can be enabled by customizing `mu4e-headers-fields' + (replacing `:subject' with `:thread-subject') + + It's not the default yet, but may become so in the future. + + - add some spam-handling actions to mu4e-contrib.el + + - mu4e now targets org 8.x, which support for previous versions + relegated to `org-old-mu4e.el`. Some of the new org-features are improved + capture templates. + + - updates to the documentation, in particular about using BBDB. + + - improved URL-handling (use emacs built-in functionality) + + - many bug fixes, including some crash fixes on BSD + +*** guile + + – add --delete option to the find-dups scripts, to automatically delete + them. Use with care! + +** Release 0.9.12 + +*** mu + + - truncate /all/ terms the go beyond xapian's max term length + - lowercase the domain-part of email addresses in mu cfind (and mu4e), if + the domain is in ascii + - give messages without msgids fake-message-ids; this fixes the problem + where such messages were not found in --include-related queries + - cleanup of the query parser + - provide fake message-ids for messages without it; fixes #183 + - allow showing tags in 'mu find' output + - fix CSV quoting + +*** mu4e + + - update the emacs <-> backend protocol; documented in the mu-server man page + - show 'None' as date for messages without it (Headers View) + - add `mu4e-headers-found-hook', `mu4e-update-pre-hook'. + - split org support in org-old-mu4e.el (org <= 7.x) and org-mu4e.el + - org: improve template keywords + - rework URL handling + +** Release 0.9.10 + +*** mu + + - allow 'contact:' as a shortcut in queries for 'from:foo OR to:foo OR + cc:foo OR bcc:foo', and 'recip:' as a shortcut for 'to:foo OR cc:foo OR + bcc:foo' + - support getting related messages (--include-related), which includes + messages that may not match the query, but that are in the same threads as + messages that were + - support "list:"/"v:" for matching mailing list names, and the "v" + format-field to show them. E.g 'mu find list:emacs-orgmode.gnu.org' + +*** mu4e + + - scroll down in message view takes you to next message (but see + `mu4e-view-scroll-to-next') + - support 'human dates', that is, show the time for today's messages, and + the date for older messages in the headers view + - replace `mu4e-user-mail-address-regexp' and `mu4e-my-mail-addresses' with + `mu4e-user-mail-address-list' + - support tags (i.e.., X-Keywords and friends) in the headers-view, and the + message view. Thanks to Abdó Roig-Maranges. New field ":tags". + - automatically update the headers buffer when new messages are found during + indexing; set `mu4e-headers-auto-update' to nil to disable this. + - update mail/index with M-x mu4e-update-mail-and-index; which everywhere in + mu4e is available with key C-S-u. Use prefix argument to run in + background. + - add function `mu4e-update-index' to only update the index + - add 'friendly-names' for mailing lists, so they should up nicely in the + headers view + +*** guile + + - add 'mu script' command to run mu script, for example to do statistics on + your message corpus. See the mu-script man-page. + +*** mug + + - ported to gtk+ 3; remove gtk+ 2.x code + + + +** Release 0.9.9 <2012-10-14> + +*** mu4e + - view: address can be toggled long/short, compose message + - sanitize opening urls (mouse-1, and not too eager) + - tooltips for header labels, flags + - add sort buttons to header-labels + - support signing / decryption of messages + - improve address-autocompletion (e.g., ensure it's case-insensitive) + - much faster when there are many maildirs + - improved line wrapping + - better handle attached messages + - improved URL-matching + - improved messages to user (mu4e-(warn|error|message)) + - add refiling functionality + - support fancy non-ascii in the UI + - dynamic folders (i.e.., allow mu4e-(sent|draft|trash|refile)-folder) to + be a function + - dynamic attachment download folder (can be a function now) + - much improved manual + +*** mu + - remove --summary (use --summary-len instead) + - add --after for mu find, to limit to messages after T + - add new command `mu verify', to verify signatures + - fix iso-2022-jp decoding (and other 7-bit clean non-ascii) + - add support for X-keywords + - performance improvements for threaded display (~ 25% for 23K msgs) + - mu improved user-help (and the 'mu help' command) + - toys/mug2 replaces toys/mug + +*** mu-guile + - automated tests + - add mu:timestamp, mu:count + - handle db reopenings in the background + + +** Release 0.9.8.5 <2012-07-01> + +*** mu4e + + - auto-completion of e-mail addresses + - inline display of images (see `mu4e-view-show-images'), uses imagemagick + if available + - interactively change number of headers / columns for showing headers with + C-+ and C-- in headers, view mode + - support flagging message + - navigate to previous/next queries like a web browser (with , + ) + - narrow search results with '/' + - next/previous take a prefix arg now, to move to the nth previous/next message + - allow for writing rich-text messages with org-mode + - enable marking messages as Flagged + - custom marker functions (see manual) + - better "dwim" handling of buffer switching / killing + - deferred marking of message (i.e.., mark now, decide what to mark for + later) + - enable changing of sort order, display of threads + - clearer marks for marked messages + - fix sorting by subject (disregarding Re:, Fwd: etc.) + - much faster handling when there are many maildirs (speedbar) + - handle mailto: links + - improved, extended documentation + +*** mu + + - support .noupdate files (parallel to .noindex, dir is ignored unless we're + doing a --rebuild). + - append all inline text parts, when getting the text body + - respect custom maildir flags + - correctly handle the case where g_utf8_strdown (str) > len (str) + - make gtk, guile, webkit dependency optional, even if they are installed + + +** Release 0.9.8.4 <2012-05-08> + +*** mu4e + + - much faster header buffers + - split view mode (headers, view); see `mu4e-split-view'. + - add search history for queries + - ability to open attachments with arbitrary programs, pipe through shell + commands or open in the current emacs + - quote names in recipient addresses + - mu4e-get-maildirs works now for recursive maildirs as well + - define arbitrary operations for headers/messages/attachments using the + actions system -- see the chapter 'Actions' in the manual + - allow mu4e to be uses as the default emacs mailer (`mu4e-user-agent') + - mark headers based on a regexp, `mu4e-mark-matches', or '%' + - mark threads, sub-threads (mu4e-hdrs-mark-thread, + mu4e-hdrs-mark-subthread, or 'T', 't') + - add msg2pdf toy + - easy logging (using `mu4e-toggle-logging') + - improve mu4e-speedbar for use in headers/view + - use the message-mode FCC system for saving messages to the sent-messages + folder + - fix: off-by-one in number of matches shown + +*** general + + - fix for opening files with non-ascii names + - much improved support for searching non-Latin (Cyrillic etc.) languages + we can now match 'Тесла' or 'Аркона' without problems + - smarter escaping (fixes issues with finding message ids) + - fixes for queries with brackets + - allow --summary-len for the length of message summaries + - numerous other small fixes + + +** Release 0.9.8.3 <2012-04-06> + + *NOTE*: existing mu/mu4e are recommended to run `mu index --rebuild' after + installation. + +*** mu4e + + - allow for searching by editing bookmarks + (`mu4e-search-bookmark-edit-first') (keybinding 'B') + - make it configurable what to do with sent messages (see + `mu4e-sent-messages-behavior') + - speedbar support (initial patch by Antono V) + - better handling of drafts: + - don't save too early + - more descriptive buffer names (based on Subject, if any) + - don't put "--text-follows-this-line--" markers in files + - automatically include signatures, if set + - add user-settable variables mu4e-view-wrap-lines and mu4e-view-hide-cited, + which determine the initial way a message is displayed + - improved documentation + +*** general + + - much improved searching for GMail folders (i.e. maildir:/ matching); + this requires a 'mu index --rebuild' + - correctly handle utf-8 messages, even if they don't specify this explicitly + - fix compiler warnings for newer/older gcc and clang/clang++ + - fix unit tests (and some code) for Ubuntu 10.04 and FreeBSD9 + - fix warnings for compilation with GTK+ 3.2 and recent glib (g_set_error) + - fix mu_msg_move_to_maildir for top-level messages + - fix in maildir scanning + - plug some memleaks + +** Release 0.9.8.2 <2012-03-11> + +*** mu4e: + + - make mail updating non-blocking + - allow for automatic periodic update ('mu4e-update-interval') + - allow for external triggering of update + - make behavior when leaving the headers buffer customizable, ie. + ask/apply/ignore ('mu4e-headers-leave-behaviour') + +*** general + + - fix output for some non-UTF8 locales + - open ('play') file names with spaces + - don't show unnecessary errors for --format=links + - make build warning-free for clang/clang++ + - allow for slightly older autotools + - fix unit tests for some hidden assumptions (locale, dir structure etc.) + - some documentation updates / clarifications + +** Release 0.9.8.1 <2012-02-18 Sat> + +*** mu + - show only leaf/rfc822 MIME-parts + +*** mu4e + + - allow for shell commands with arguments in `mu4e-get-mail-command'. + - support marking messages as 'read' and 'unread' + - show the current query in the the mode-line (`global-mode-string'). + - don't repeat 'Re:' / 'Fwd:' + - colorize cited message parts + - better handling of text-based, embedded message attachments + - for text-bodies, concatenate all text/plain parts + - make filladapt dep optional + - documentation improvements + +** Release 0.9.8 <2012-01-31> + + - '--descending' has been renamed into '--reverse' + - search for attachment MIME-type using 'mime:' or 'y:' + - search for text in text-attachments using 'embed:' or 'e:' + - searching for attachment file names now uses 'file:' (was: 'attach:') + - experimental emacs-based mail client -- "mu4e" + - added more unit tests + - improved guile binding - no special binary is needed anymore, it's + installable are works with the normal guile system; code has been + substantially improved. still 'experimental' + +** Release 0.9.7 <2011-09-03 Sat> + + - don't enforce UTF-8 output, use locale (fixes issue #11) + - add mail threading to mu-find (using -t/--threads) (sorta fixes issue #13) + - add header line to --format=mutt-ab (mu cfind), (fixes issue #42) + - terminate mu view results with a form-feed marker (use --terminate) (fixes + issue #41) + - search X-Label: tags (fixes issue #40) + - added toys/muile, the mu guile shells, which allows for message stats etc. + - fix date handling (timezones) + +** Release 0.9.6 <2011-05-28 Sat> + + - FreeBSD build fix + - fix matching for mu cfind to be as expected + - fix mu-contacts for broken names/emails + - clear the contacts-cache too when doing a --rebuild + - wildcard searches ('*') for fields (except for path/maildir) + - search for attachment file names (with 'a:'/'attach:') -- also works with + wildcards + - remove --xquery completely; use --output=xquery instead + - fix progress info in 'mu index' + - display the references for a message using the 'r' character (xmu find) + - remove --summary-len/-k, instead use --summary for mu view and mu find, and + - support colorized output for some sub-commands (view, cfind and + extract). Disabled by default, use --color to enable, or set env MU_COLORS + to non-empty + - update documentation, added more examples + +** Release 0.9.5 <2011-04-25 Mon> + + - bug fix for infinite loop in Maildir detection + - minor fixes in tests, small optimizations + +** Release 0.9.4 <2011-04-12 Tue> + + - add the 'cfind' command, to search/export contact information + - add 'flag:unread' as a synonym for 'flag:new OR NOT flag:unseen' + - updated documentation + +** Release 0.9.3 <2011-02-13 Sun> + + - don't warn about missing files with --quiet + +** Release 0.9.2 <2011-02-02 Wed> + + - stricter checking of options; and options must now *follow* the sub-command + (if any); so, something like: 'mu index --maildir=/foo/bar' + - output searches as plain text (default), XML, JSON or s-expressions using + --format=plain|xml|json|sexp. For example: 'mu find foobar --output=json'. + These format options are experimental (except for 'plain') + - the --xquery option should now be used as --format=xquery, for output + symlinks, use --format=links. This is a change in the options. + - search output can include the message size using the 'z' shortcut + - match message size ranges (i.e.. size:500k..2M) + - fix: honor the --overwrite (or lack thereof) parameter + - support folder names with special characters (@, ' ', '.' and so on) + - better check for already-running mu index + - when --maildir= is not provided for mu index, default to the last one + - add --max-msg-size, to specify a new maximum message size + - move the 'mug' UI to toys/mug; no longer installable + - better support for Solaris builds, Gentoo. + +** Release 0.9.1 <2010-12-05 Sun> + + - Add missing icon for mug + - Fix unit tests (Issue #30) + - Fix Fedora 14 build (broken GTK+ 3) (Issue #31) + +** Release 0.9 <2010-12-04 Sat> + + - you can now search for the message priority ('prio:high', 'prio:low', + 'prio:normal') + - you can now search for message flags, e.g. 'flag:attach' for messages with + attachment, or 'flag:encrypted' for encrypted messages + - you can search for time-intervals, e.g. 'date:2010-11-26..2010-11-29' for + messages in that range. See the mu-find(1) and mu-easy(1) man-pages for + details and examples. + - you can store bookmarked queries in ~/.mu/bookmarks + - the 'flags' parameter has been renamed in 'flag' + - add a simple graphical UI for searching, called 'mug' + - fix --clearlinks for file systems without entry->d_type (fixes issue #28) + - make matching case-insensitive and accent-insensitive (accent-insensitive + for characters in Unicode Blocks 'Latin-1 Supplement' and 'Latin + Extended-A') + - more extensive pre-processing is done to make searching for email-addresses + and message-ids less likely to not work (issue #21) + - updated the man-pages + - experimental support for Fedora 14, which uses GMime 2.5.x (fixes issue #29) + +** Release 0.8 <2010-10-30 Sat> + + - There's now 'mu extract' for getting information about MIME-parts + (attachments) and extracting them + - Queries are now internally converted to lowercase; this solves some of the + false-negative issues + - All mu sub-commands now have their own man-page + - 'mu find' now takes a --summary-len= argument to print a summary of + up-to-n lines of the message + - Same for 'mu view'; the summary replaces the full body + - Setting the mu home dir now goes with -m, --muhome + - --log-stderr, --reindex, --rebuild, --autoupgrade, --nocleanup, --mode, + --linksdir, --clearlinks lost their single char version + +** Release 0.7 <2010-02-27 Sat> + + - Database format changed + - Automatic database scheme version check, notifies users when an upgrade + is needed + - 'mu view', to view mail message files + - Support for >10K matches + - Support for unattended upgrades - that is, the database can automatically + by upgraded (--autoupgrade). Also, the log file is automatically cleaned + when it gets too big (unless you use --nocleanup) + - Search for a certain Maildir using the maildir:,m: search prefixes. For + example, you can find all messages located in ~/Maildir/foo/bar/cur/msg + ~/Maildir/foo/bar/new/msg and with m:/foo/bar this replace the search for + path/p in 0.6 + - Fixes for reported issues () + - A test suite with a growing number of unit tests + + +** Release 0.6 <2010-01-23 Sat> + + - First new release of mu since 2008 + - No longer depends on sqlite + + +# Local Variables: +# mode: org; org-startup-folded: nil +# fill-column:80 +# End: diff --git a/README.org b/README.org new file mode 100644 index 0000000..ec5c648 --- /dev/null +++ b/README.org @@ -0,0 +1,104 @@ +#+TITLE:mu +[[https://github.com/djcb/mu/blob/master/COPYING][https://img.shields.io/github/license/djcb/mu?logo=gnu&.svg]] +[[https://en.cppreference.com][https://img.shields.io/badge/Made%20with-C/CPP-1f425f?logo=c&.svg]] +[[https://img.shields.io/github/v/release/djcb/mu][https://img.shields.io/github/v/release/djcb/mu.svg]] +[[https://github.com/djcb/mu/graphs/contributors][https://img.shields.io/github/contributors/djcb/mu.svg]] +[[https://github.com/djcb/mu/issues][https://img.shields.io/github/issues/djcb/mu.svg]] +[[https://github.com/djcb/mu/issues?q=is%3Aissue+is%3Aopen+label%3Arfe][https://img.shields.io/github/issues/djcb/mu/rfe?color=008b8b.svg]] +[[https://github.com/djcb/mu/pull/new][https://img.shields.io/badge/PRs-welcome-brightgreen.svg]]\\ +[[https://www.gnu.org/software/emacs/][https://img.shields.io/badge/Emacs-26.3-922793?logo=gnu-emacs&logoColor=b39ddb&.svg]] +[[https://www.djcbsoftware.nl/code/mu/mu4e/Installation.html#Dependencies-for-Debian_002fUbuntu][https://img.shields.io/badge/Platform-Linux-2e8b57?logo=linux&.svg]] +[[https://www.djcbsoftware.nl/code/mu/mu4e/Installation.html#Building-from-a-release-tarball-1][https://img.shields.io/badge/Platform-FreeBSD-8b3a3a?logo=freebsd&logoColor=c32136&.svg]] +[[https://formulae.brew.sh/formula/mu#default][https://img.shields.io/badge/Platform-macOS-101010?logo=apple&logoColor=ffffff&.svg]] + + [ *Note*: you are looking at the *development* branch, which is where new code is + being developed and tested, and which may occasionally break. Distributions and + non-adventurous users are instead recommended to use the [[https://github.com/djcb/mu/tree/release/1.10][1.10 Release Branch]] or + to pick up one of the [[https://github.com/djcb/mu/releases][1.10 Releases]]. ] + +Welcome to ~mu~! + +Latest development news: [[NEWS.org]]. + +With the enormous amounts of e-mail many people gather and the importance of +e-mail message in our work-flows, it's essential to quickly deal with all that +mail - in particular, to instantly find that one important e-mail you need right +now, and quickly file away message for later use. + +~mu~ is a tool for dealing with e-mail messages stored in the Maildir-format. ~mu~'s +purpose in life is to help you to quickly find the messages you need; in +addition, it allows you to view messages, extract attachments, create new +maildirs, and so on. + +After indexing your messages into a [[http://www.xapian.org][Xapian]]-database, you can search them using a +custom query language. You can use various message fields or words in the body +text to find the right messages. + +Built on top of ~mu~ are some extensions (included in this package): + +- ~mu4e~: a full-featured e-mail client that runs inside emacs + +- ~mu-guile~: bindings for the Guile/Scheme programming language (version 3.0 and + later) + +~mu~ is written in C++; ~mu4e~ is written in ~elisp~ and ~mu-guile~ in a mix of C++ and +Scheme. + +~mu~ is available in Linux distributions (e.g. Debian/Ubuntu and Fedora) under the +name ~maildir-utils~; apparently because they don't like short names. All of the +code is distributed under the terms of the [[https://www.gnu.org/licenses/gpl-3.0.en.html][GNU General Public License version 3]] +(or higher). + +* Installation + +Note: building from source is an /advanced/ subject, especially if something goes +wrong. The below simple examples are a start, but all tools involved have many +options; there are differences between systems, versions etc. So if this is all +a bit daunting we recommend to wait for someone else to build it for you, such +as a Linux distribution. Many have packages available. + +** Requirements + +To be able to build ~mu~, ensure you have: + +- a C++17 compiler (~gcc~ or ~clang~ are known to work) +- development packages for /Xapian/ and /GMime/ and /GLib/ (see ~meson.build~ for thex + versions) +- basic tools such as ~make~, ~sed~, ~grep~ +- ~meson~ + +For ~mu4e~, you also need ~emacs~. + +Note, support for Windows is very much _experimental_, that is, it works for some +people, but we can't really support it due to lack of the specific expertise. +Help is welcome! + +** Building + +#+begin_example +$ git clone git://github.com/djcb/mu.git +$ cd mu +#+end_example + +~mu~ uses ~meson~ for building, so you can use that directly, and all the usual +commands apply. You can also use it _indirectly_ through the provided ~Makefile~, +which provides a number of useful targets. + +For instance, using the ~Makefile~, you could install ~mu~ using: + +#+begin_example +$ ./autogen.sh && make +$ sudo make install +#+end_example + +Alternatively, you can run ~meson~ directly (see the ~meson~ documentation for +more details): +#+begin_example +$ meson setup -C build +$ meson compile -C build +$ meson install -C build +#+end_example + +** Contributing + +Contributions are welcome! See the Github issue list and [[IDEAS.org]]. diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..a0eabf8 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +echo "*** meson build setup" + +test -f mu/mu.cc || { + echo "*** Run this script from the top-level mu source directory" + exit 1 +} +BUILDDIR=build + +command -v meson 2> /dev/null +if [ $? != 0 ]; then + echo "*** 'meson' not found, please install it ***" + exit 1 +fi + +# we could remove build/ but let's avoid rm -rf risks... +if test -d ${BUILDDIR}; then + meson setup --reconfigure ${BUILDDIR} $@ || exit 1 +else + meson setup ${BUILDDIR} $@ || exit 1 +fi + +echo "*** Now run either 'ninja -C ${BUILDDIR}' or 'make' to build mu" +echo "*** Check the Makefile for other useful targets" diff --git a/build-aux/date.py b/build-aux/date.py new file mode 100755 index 0000000..d93b13d --- /dev/null +++ b/build-aux/date.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +""" +Script to get date strings, since the MacOS 'date' is not quite up to GNU +standards + +E.g.. + date.py 2023-10-14 "The year-month is %y %m" +""" + +import sys +from datetime import datetime + +date=datetime.strptime(sys.argv[1],'%Y-%m-%d') +print(date.strftime(sys.argv[2])) diff --git a/build-aux/meson-install-info.sh b/build-aux/meson-install-info.sh new file mode 100644 index 0000000..0019249 --- /dev/null +++ b/build-aux/meson-install-info.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +infodir=$1 +infofile=$2 + +# Meson post-install script to update info metadata + +# If DESTDIR is set, do _not_ install-info, since it's only a temporary +# install +if test -z "${DESTDIR}"; then + install-info --info-dir "${infodir}" "${infodir}/${infofile}" + gzip --best --force "${infodir}/${infofile}" +fi diff --git a/build-aux/version.texi.in b/build-aux/version.texi.in new file mode 100644 index 0000000..aa13bab --- /dev/null +++ b/build-aux/version.texi.in @@ -0,0 +1,5 @@ +@set UPDATED @UPDATED@ +@set UPDATED-MONTH @UPDATEDMONTH@ +@set UPDATED-YEAR @UPDATEDYEAR@ +@set EDITION @VERSION@ +@set VERSION @VERSION@ diff --git a/contrib/mu-completion.zsh b/contrib/mu-completion.zsh new file mode 100644 index 0000000..ea2bdbd --- /dev/null +++ b/contrib/mu-completion.zsh @@ -0,0 +1,124 @@ +#compdef mu + +## Copyright (C) 2011-2012 Dirk-Jan C. Binnema +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# zsh completion for mu. Install this by copying/linking to this file somewhere in +# your $fpath; the link/copy must have a name starting with an underscore "_" + +# main dispatcher function +_mu() { + if (( CURRENT > 2 )) ; then + local cmd=${words[2]} + curcontext="${curcontext%:*:*}:mu-$cmd" + (( CURRENT-- )) + shift words + _call_function ret _mu_$cmd + return ret + else + _mu_commands + fi +} + + + +_mu_commands() { + local -a mu_commands + mu_commands=( + 'index:scan your maildirs and import their metadata in the database' + 'find:search for messages in the database' + 'view:display specific messages' + 'cfind:search for contacts (name + email) in the database' + 'extract:extract message-parts (attachments) and save or open them' + 'mkdir:create maildirs' +# below are not generally very useful, so let's not auto-complete them +# 'add: add a message to the database.' +# 'remove:remove a message from the database.' +# 'server:sart the mu server' +) + + _describe -t command 'command' mu_commands +} + +_mu_common_options=( + '--debug[output information useful for debugging mu]' + '--quiet[do not give any non-critical information]' + '--nocolor[do not use colors in some of the output]' + '--version[display mu version and copyright information]' + '--log-stderr[log to standard error]' +) + +_mu_db_options=( + '--muhome[use some non-default location for the mu database]:directory:_files' +) + +_mu_find_options=( + '--fields[fields to display in the output]' + '--sortfield[field to sort the output by]' + '--descending[sort in descending order]' + '--summary[include a summary of the message]' + '--summary-len[number of lines to use for the summary]' + '--bookmark[use a named bookmark]' + '--output[set the kind of output for the query]' +) + +_mu_view_options=( + '--summary[only show a summary of the message]' + '--summary-len[number of lines to use for the summary]' +) + + +_mu_view() { + _arguments -s : \ + $_mu_common_options \ + $_mu_view_options +} + +_mu_extract() { + _files +} + +_mu_find() { + _arguments -s : \ + $_mu_common_options \ + $_mu_db_options \ + $_mu_find_options +} + +_mu_index() { + _arguments -s : \ + $_mu_db_options \ + $_mu_common_options +}mu + +_mu_cleanup() { + _arguments -s : \ + $_mu_db_options \ + $_mu_common_options +} + + +_mu_mkdir() { + _arguments -s : \ + '--mode=[file mode for the new Maildir]:file mode: ' \ + $_mu_common_options +} + +_mu "$@" + +# Local variables: +# mode: sh +# End: diff --git a/contrib/mu-sexp-convert b/contrib/mu-sexp-convert new file mode 100755 index 0000000..b2835ac --- /dev/null +++ b/contrib/mu-sexp-convert @@ -0,0 +1,204 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; Copyright (C) 2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +;; +;; a little hack to convert the output of +;; mu find --format=sexp +;; and +;; mu view --format=sexp +;; into XML or JSON + +(use-modules (ice-9 getopt-long) (ice-9 format) (ice-9 regex)) +(use-modules (sxml simple)) + +(define (mapconcat func lst sepa) + "Apply FUNC to elements of LST, concat the result as strings +separated by SEPA." + (if (null? lst) + "" + (string-append + (func (car lst)) + (if (null? (cdr lst)) + "" + (string-append sepa (mapconcat func (cdr lst) sepa)))))) + +(define (property-list? obj) + "Is OBJ a elisp-style property list (ie. a list of the +form (:symbol1 something :symbol2 somethingelse), as in an elisp +proplilst." + (and (list? obj) + (not (null? obj)) + (symbol? (car obj)) + (string= ":" (substring (symbol->string (car obj)) 0 1)))) + +(define (plist->pairs plist) + "Convert an elisp-style property list; e.g: + (:prop1 foo :prop2: bar ...) +into a list of pairs + ((prop1 . foo) (prop2 . bar) ...)." + (if (null? plist) + '() + (cons + (cons + (substring (symbol->string (car plist)) 1) + (cadr plist)) + (plist->pairs (cddr plist))))) + +(define (string->xml str) + "XML-encode STR." + ;; sneakily re-using sxml->xml + (call-with-output-string (lambda (port) (sxml->xml str port)))) + +(define (string->json str) + "Convert string into a JSON-encoded string." + (letrec ((convert + (lambda (lst) + (if (null? lst) + "" + (string-append + (cond + ((equal? (car lst) #\") "\\\"") + ((equal? (car lst) #\\) "\\\\") + ((equal? (car lst) #\/) "\\/") + ((equal? (car lst) #\bs) "\\b") + ((equal? (car lst) #\ff) "\\f") + ((equal? (car lst) #\lf) "\\n") + ((equal? (car lst) #\cr) "\\r") + ((equal? (car lst) #\ht) "\\t") + (#t (string (car lst)))) + (convert (cdr lst))))))) + (convert (string->list str)))) + +(define (etime->time_t t) + "Convert elisp time object T into a time_t value." + (logior (ash (car t) 16) (car (cdr t)))) + +(define (sexp->xml) + "Convert string INPUT to XML, return the XML (string)." + (letrec ((convert-xml + (lambda* (expr #:optional parent) + (cond + ((property-list? expr) + (mapconcat + (lambda (pair) + (format #f "\t<~a>~a\n" + (car pair) (convert-xml (cdr pair) (car pair)) (car pair))) + (plist->pairs expr) " ")) + ((list? expr) + (cond + ((member parent '("from" "to" "cc" "bcc")) + (mapconcat (lambda (addr) + (format #f "
~a~a
" + (if (string? (car addr)) + (format #f "~a" + (string->xml (car addr))) "") + (if (string? (cdr addr)) + (format #f "~a" + (string->xml (cdr addr))) ""))) + expr " ")) + ((string= parent "parts") "") ;; for now, ignore + ;; convert the crazy emacs time thingy to time_t... + ((string= parent "date") (format #f "~a" (etime->time_t expr))) + (#t + (mapconcat + (lambda (elm) (format #f "~a" (convert-xml elm))) expr "")))) + ((string? expr) (string->xml expr)) + ((symbol? expr) (format #f "~a" expr)) + ((number? expr) (number->string expr)) + (#t ".")))) + (msg->xml + (lambda () + (let ((expr (read))) + (if (not (eof-object? expr)) + (string-append (format #f "\n~a\n" (convert-xml expr)) (msg->xml)) + ""))))) + (format #f "\n\n~a" (msg->xml)))) + + +(define (sexp->json) + "Convert string INPUT to JSON, return the JSON (string)." + (letrec ((convert-json + (lambda* (expr #:optional parent) + (cond + ((property-list? expr) + (mapconcat + (lambda (pair) + (format #f "\n\t\"~a\": ~a" + (car pair) (convert-json (cdr pair) (car pair)))) + (plist->pairs expr) ", ")) + ((list? expr) + (cond + ((member parent '("from" "to" "cc" "bcc")) + (string-append "[" + (mapconcat (lambda (addr) + (format #f "{~a~a}" + (if (string? (car addr)) + (format #f "\"name\": \"~a\"," + (string->json (car addr))) "") + (if (string? (cdr addr)) + (format #f "\"email\": \"~a\"" + (string->json (cdr addr))) ""))) + expr ", ") + "]")) + ((string= parent "parts") "[]") ;; todo + ;; convert the crazy emacs time thingy to time_t... + ((string= parent "date") + (format #f "~a" (format #f "~a" (etime->time_t expr)))) + (#t + (string-append "[" + (mapconcat (lambda (elm) (format #f "~a" (convert-json elm))) expr ",") "]")))) + ((string? expr) + (format #f "\"~a\"" (string->json expr))) + ((symbol? expr) + (format #f "\"~a\"" expr)) + ((number? expr) (number->string expr)) + (#t ".")))) + (msg->json + (lambda (first) + (let ((expr (read))) + (if (not (eof-object? expr)) + (string-append (format #f "~a{~a\n}" + (if first "" ",\n") + (convert-json expr)) (msg->json #f)) + ""))))) + (format #f "[\n~a\n]" (msg->json #t)))) + +(define (main args) + (let* ((optionspec '((format (value #t)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: mu-sexp-convert " + "--format=\n" + "reads from standard-input and prints to standard output\n")) + (outformat (or (option-ref options 'format #f) + (begin (display msg) (exit 1))))) + (cond + ((string= outformat "xml") + (format #t "~a\n" (sexp->xml))) + ((string= outformat "json") + (format #t "~a\n" (sexp->json))) + (#t (begin + (display msg) + (exit 1)))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/contrib/mu.spec b/contrib/mu.spec new file mode 100644 index 0000000..1ed66d4 --- /dev/null +++ b/contrib/mu.spec @@ -0,0 +1,129 @@ + +# These refer to the release version +# When 0.9.9.6 gets out, remove the global pre line +%global pre pre2 +%global rel 1 + +Summary: A lightweight email search engine for Maildirs +Name: mu +Version: 0.9.9.6 +URL: https://github.com/djcb/mu +# From Packaging:NamingGuidelines for pre-relase versions: +# Release: 0.%{X}.%{alphatag} where %{X} is the release number +%if %{pre} +Release: 0.%{rel}.%{prerelease}%{?dist} +%else +Release: %{rel}%{?dist} +%endif + +License: GPLv3 +Group: Applications/Internet +BuildRoot: %{_tmppath}/%{name}-%{version}-build + +# Source is at ssaavedra repo because djcb has not yet this version tag created +Source0: http://github.com/ssaavedra/%{name}/archive/v%{version}%{?pre}.tar.gz +BuildRequires: emacs-el +BuildRequires: emacs +BuildRequires: gmime-devel +BuildRequires: guile-devel +BuildRequires: xapian-core-devel +BuildRequires: libuuid-devel +BuildRequires: texinfo +Requires: gmime +Requires: guile +Requires: xapian-core-libs +Requires: emacs-filesystem >= %{_emacs_version} + + +%description +E-mail is the 'flow' in the work flow of many people. Consequently, one spends a lot of time searching for old e-mails, to dig up some important piece of information. With people having tens of thousands of e-mails (or more), this is becoming harder and harder. How to find that one e-mail in an ever-growing haystack? +Enter mu. +'mu' is a set of command-line tools for Linux/Unix that enable you to quickly find the e-mails you are looking for, assuming that you store your e-mails in Maildirs (if you don't know what 'Maildirs' are, you are probably not using them). + +%package gtk +Group: Applications/Internet +Summary: GUI for using mu (called mug) +BuildRequires: gtk3-devel +BuildRequires: webkitgtk3-devel +Requires: gtk3 +Requires: gmime +Requires: webkitgtk3 +Requires: mu = %{version}-%{release} + +%description gtk +Mug is a simple GUI for mu from version 0.9. + +%package guile +Group: Applications/Internet +Summary: Guile scripting capabilities for mu +Requires: guile +Requires: mu = %{version}-%{release} +Requires(post): info +Requires(preun): info + +%description guile +Bindings for Guile to interact with mu. + + +%prep +%setup -n %{name}-%{version}%{?pre} -q + +%build +autoreconf -i +%configure +make %{?_smp_mflags} + +%install +rm -rf %{buildroot} +make install DESTDIR=%{buildroot} +install -p -c -m 755 %{_builddir}/%{buildsubdir}/toys/mug/mug %{buildroot}%{_bindir}/mug +cp -p %{_builddir}/%{buildsubdir}/mu4e/*.el %{buildroot}%{_emacs_sitelispdir}/mu4e/ +rm -f %{buildroot}%{_infodir}/dir + +%clean +rm -rf %{buildroot} + +%post +/sbin/install-info \ + --info-dir=%{_infodir} %{_infodir}/mu4e.info.gz || : +%preun +if [ $1 = 0 -a -f %{_infodir}/mu4e.info.gz ]; then + /sbin/install-info --delete \ + --info-dir=%{_infodir} %{_infodir}/mu4e.info.gz || : +fi + +%post guile +/sbin/install-info \ + --info-dir=%{_infodir} %{_infodir}/mu-guile.info.gz || : + +%preun guile +if [ $1 = 0 -a -f %{_infodir}/mu-guile.info.gz ]; then + /sbin/install-info --delete \ + --info-dir=%{_infodir} %{_infodir}/mu-guile.info.gz || : +fi + + +%files +%defattr(-,root,root) +%{_bindir}/mu +%{_mandir}/man1/* +%{_mandir}/man5/* +%{_datadir}/mu/* + +%{_emacs_sitelispdir}/mu4e +%{_emacs_sitelispdir}/mu4e/*.elc +%{_emacs_sitelispdir}/mu4e/*.el +%{_infodir}/mu4e.info.gz + +%files gtk +%{_bindir}/mug + +%files guile +%{_libdir}/libguile-mu.* +%{_datadir}/guile/site/2.0/mu/* +%{_datadir}/guile/site/2.0/mu.scm +%{_infodir}/mu-guile.info.gz + +%changelog +* Wed Feb 12 2014 Santiago Saavedra - 0.9.9.5-1 +- Create first SPEC. diff --git a/guile/compile-scm.in b/guile/compile-scm.in new file mode 100644 index 0000000..04cc0f9 --- /dev/null +++ b/guile/compile-scm.in @@ -0,0 +1,22 @@ +#!/bin/sh +## Copyright (C) 2021 Dirk-Jan C. Binnema +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +@abs_builddir@/build-env @guild@ compile "$@" + +# Local-Variables: +# mode: sh +# End: diff --git a/guile/examples/contacts-export b/guile/examples/contacts-export new file mode 100755 index 0000000..7e33c54 --- /dev/null +++ b/guile/examples/contacts-export @@ -0,0 +1,85 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; +;; Copyright (C) 2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +(use-modules (ice-9 getopt-long) (ice-9 format)) +(use-modules (srfi srfi-1)) +(use-modules (mu)) + +(define (sort-by-freq c1 c2) + (< (mu:frequency c1) (mu:frequency c2))) + +(define (sort-by-newness c1 c2) + (< (mu:last-seen c1) (mu:last-seen c2))) + +(define (main args) + (let* ((optionspec '( (muhome (value #t)) + (sort-by (value #t)) + (revert (value #f)) + (format (value #t)) + (limit (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: contacts-export [--help] [--muhome=] " + "--format= " + "--sort-by= [--revert] [--limit=]\n")) + (help (option-ref options 'help #f)) + (muhome (option-ref options 'muhome #f)) + (sort-by (or (option-ref options 'sort-by #f) "frequency")) + (revert (option-ref options 'revert #f)) + (form (or (option-ref options 'format #f) "plain")) + (limit (string->number (option-ref options 'limit "1000000")))) + (if help + (begin + (display msg) + (exit 0)) + (begin + (setlocale LC_ALL "") + (mu:initialize muhome) + (let* ((sort-func + (cond + ((string= sort-by "frequency") sort-by-freq) + ((string= sort-by "newness") sort-by-newness) + (else (begin (display msg) (exit 1))))) + (contacts '())) + ;; make a list of all contacts + (mu:for-each-contact + (lambda (c) (set! contacts (cons c contacts)))) + + ;; should we sort it? + (if sort-by + (set! contacts (sort! contacts + (if revert (negate sort-func) sort-func)))) + + ;; should we limit the number? + (if (and limit (< limit (length contacts))) + (set! contacts (take! contacts limit))) + ;; export! + (for-each + (lambda (c) + (format #t "~a\n" (mu:contact->string c form))) + contacts)))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/examples/msg-graphs b/guile/examples/msg-graphs new file mode 100755 index 0000000..654dd28 --- /dev/null +++ b/guile/examples/msg-graphs @@ -0,0 +1,133 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# +;; +;; Copyright (C) 2011-2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +(setlocale LC_ALL "") + +(use-modules (ice-9 getopt-long) (ice-9 optargs) (ice-9 popen) (ice-9 format)) +(use-modules (mu) (mu stats) (mu plot)) +;;(use-modules (mu) (mu message) (mu stats) (mu plot)) + +(define (per-hour expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (sort + (mu:tabulate + (lambda (msg) + (tm:hour (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per hour matching ~a" expr) "Hour" "Messages" output)) + +(define (per-day expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (mu:weekday-numbers->names + (sort (mu:tabulate + (lambda (msg) + (tm:wday (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y))))) + (format #f "Messages per weekday matching ~a" expr) "Day" "Messages" output)) + +(define (per-month expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (mu:month-numbers->names + (sort + (mu:tabulate + (lambda (msg) + (tm:mon (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y))))) + (format #f "Messages per month matching ~a" expr) "Month" "Messages" output)) + + +(define (per-year-month expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (sort (mu:tabulate + (lambda (msg) + (string->number + (format #f "~d~2'0d" + (+ 1900 (tm:year (localtime (mu:date msg)))) + (tm:mon (localtime (mu:date msg)))))) + expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per year/month matching ~a" expr) + "Year/Month" "Messages" output)) + + + +(define (per-year expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (sort (mu:tabulate + (lambda (msg) + (+ 1900 (tm:year (localtime (mu:date msg))))) expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per year matching ~a" expr) "Year" "Messages" output)) + + + +(define (main args) + (let* ((optionspec '( (muhome (value #t)) + (what (value #t)) + (text (value #f)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: mu-msg-stats [--help] [--text] " + "[--muhome=] " + "--what= [searchexpr]\n")) + (help (option-ref options 'help #f)) + (what (option-ref options 'what #f)) + (text (option-ref options 'text #f)) + ;; if `text' is `#f', use a graphical window by setting output to "wxt", + ;; else use text-mode plotting ("dumb") + (output (if text "dumb" "wxt")) + (muhome (option-ref options 'muhome #f)) + (restargs (option-ref options '() #f)) + (expr (if restargs (string-join restargs) ""))) + (if (or help (not what)) + (begin + (display msg) + (exit (if help 0 1)))) + (mu:initialize muhome) + (cond + ((string= what "per-hour") (per-hour expr output)) + ((string= what "per-day") (per-day expr output)) + ((string= what "per-month") (per-month expr output)) + ((string= what "per-year-month") (per-year-month expr output)) + ((string= what "per-year") (per-year expr output)) + (else (begin + (display msg) + (exit 1)))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/examples/mu-biff b/guile/examples/mu-biff new file mode 100755 index 0000000..bc6d507 --- /dev/null +++ b/guile/examples/mu-biff @@ -0,0 +1,59 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; +;; Copyright (C) 2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +;; script to list the message matching which are newer than +;; minutes + +;; use it, eg. like: +;; $ mu-biff --newer-than=`date +%s --date='5 minutes ago'` "maildir:/inbox" + + +(use-modules (ice-9 getopt-long) (ice-9 format)) +(use-modules (mu)) + +(define (main args) + (let* ((optionspec '((muhome (value #t)) + (newer-than (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: mu-biff [--help] [--muhome=]" + " [--newer-than=] ")) + (help (option-ref options 'help #f)) + (newer-than (string->number (option-ref options 'newer-than "0"))) + (muhome (option-ref options 'muhome #f)) + (query (string-concatenate (option-ref options '() '())))) + (if help + (begin (display msg) (newline) (exit 0)) + (begin + (mu:initialize muhome) + (mu:for-each-message + (lambda (msg) + (if (> (mu:timestamp msg) newer-than) + (format #t "~a ~a\n" + (mu:from msg) + (mu:subject msg)))) + query))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/examples/org2mu4e b/guile/examples/org2mu4e new file mode 100755 index 0000000..3556b9a --- /dev/null +++ b/guile/examples/org2mu4e @@ -0,0 +1,78 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; +;; Copyright (C) 2011-2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +(use-modules (ice-9 getopt-long) (ice-9 format)) +(use-modules (mu)) + +(define (display-org-header query) + "Print the header for the org-file for QUERY." + (format #t "* Messages matching '~a'\n\n" query)) + +(define (org-mu4e-link msg) + "Create a link for this message understandable by org-mu4e." + (let* ((subject ;; cleanup subject + (string-map + (lambda (kar) + (if (member kar '(#\] #\[)) #\space kar)) + (or (mu:subject msg) "No subject")))) + (format #f "[[mu4e:msgid:~a][~s]]" + (mu:message-id msg) subject))) + +(define (display-org-entry msg tag) + "Write an org entry for MSG." + (format #t "** ~a ~a\n\t~s\n\t~s\n" + (org-mu4e-link msg) + (if tag (string-concatenate `(":" ,tag "::")) "") + (or (mu:from msg) "?") + (let ((body (mu:body-txt msg))) + (if (not body) ;; get a 'summary' of the body text + "" + (string-map + (lambda (c) + (if (or (char=? c #\newline) (char=? c #\return)) + #\space + c)) + (substring body 0 (min (string-length body) 100))))))) + +(define (main args) + (let* ((optionspec '( (muhome (value #t)) + (tag (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: mu4e-org [--help] [--muhome=] [--tag=] ")) + (help (option-ref options 'help #f)) + (tag (option-ref options 'tag #f)) + (muhome (option-ref options 'muhome #f)) + (query (string-concatenate (option-ref options '() '())))) + (if help + (begin (display msg) (exit 0)) + (begin + (mu:initialize muhome) + (display-org-header query) + (mu:for-each-message + (lambda (msg) (display-org-entry msg tag)) + query))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/fdl.texi b/guile/fdl.texi new file mode 100644 index 0000000..96ce74e --- /dev/null +++ b/guile/fdl.texi @@ -0,0 +1,451 @@ +@c The GNU Free Documentation License. +@center Version 1.2, November 2002 + +@c This file is intended to be included within another document, +@c hence no sectioning command or @node. + +@display +Copyright @copyright{} 2000,2001,2002 Free Software Foundation, Inc. +51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. +@end display + +@enumerate 0 +@item +PREAMBLE + +The purpose of this License is to make a manual, textbook, or other +functional and useful document @dfn{free} in the sense of freedom: to +assure everyone the effective freedom to copy and redistribute it, +with or without modifying it, either commercially or noncommercially. +Secondarily, this License preserves for the author and publisher a way +to get credit for their work, while not being considered responsible +for modifications made by others. + +This License is a kind of ``copyleft'', which means that derivative +works of the document must themselves be free in the same sense. It +complements the GNU General Public License, which is a copyleft +license designed for free software. + +We have designed this License in order to use it for manuals for free +software, because free software needs free documentation: a free +program should come with manuals providing the same freedoms that the +software does. But this License is not limited to software manuals; +it can be used for any textual work, regardless of subject matter or +whether it is published as a printed book. We recommend this License +principally for works whose purpose is instruction or reference. + +@item +APPLICABILITY AND DEFINITIONS + +This License applies to any manual or other work, in any medium, that +contains a notice placed by the copyright holder saying it can be +distributed under the terms of this License. Such a notice grants a +world-wide, royalty-free license, unlimited in duration, to use that +work under the conditions stated herein. The ``Document'', below, +refers to any such manual or work. Any member of the public is a +licensee, and is addressed as ``you''. You accept the license if you +copy, modify or distribute the work in a way requiring permission +under copyright law. + +A ``Modified Version'' of the Document means any work containing the +Document or a portion of it, either copied verbatim, or with +modifications and/or translated into another language. + +A ``Secondary Section'' is a named appendix or a front-matter section +of the Document that deals exclusively with the relationship of the +publishers or authors of the Document to the Document's overall +subject (or to related matters) and contains nothing that could fall +directly within that overall subject. (Thus, if the Document is in +part a textbook of mathematics, a Secondary Section may not explain +any mathematics.) The relationship could be a matter of historical +connection with the subject or with related matters, or of legal, +commercial, philosophical, ethical or political position regarding +them. + +The ``Invariant Sections'' are certain Secondary Sections whose titles +are designated, as being those of Invariant Sections, in the notice +that says that the Document is released under this License. If a +section does not fit the above definition of Secondary then it is not +allowed to be designated as Invariant. The Document may contain zero +Invariant Sections. If the Document does not identify any Invariant +Sections then there are none. + +The ``Cover Texts'' are certain short passages of text that are listed, +as Front-Cover Texts or Back-Cover Texts, in the notice that says that +the Document is released under this License. A Front-Cover Text may +be at most 5 words, and a Back-Cover Text may be at most 25 words. + +A ``Transparent'' copy of the Document means a machine-readable copy, +represented in a format whose specification is available to the +general public, that is suitable for revising the document +straightforwardly with generic text editors or (for images composed of +pixels) generic paint programs or (for drawings) some widely available +drawing editor, and that is suitable for input to text formatters or +for automatic translation to a variety of formats suitable for input +to text formatters. A copy made in an otherwise Transparent file +format whose markup, or absence of markup, has been arranged to thwart +or discourage subsequent modification by readers is not Transparent. +An image format is not Transparent if used for any substantial amount +of text. A copy that is not ``Transparent'' is called ``Opaque''. + +Examples of suitable formats for Transparent copies include plain +@sc{ascii} without markup, Texinfo input format, La@TeX{} input +format, @acronym{SGML} or @acronym{XML} using a publicly available +@acronym{DTD}, and standard-conforming simple @acronym{HTML}, +PostScript or @acronym{PDF} designed for human modification. Examples +of transparent image formats include @acronym{PNG}, @acronym{XCF} and +@acronym{JPG}. Opaque formats include proprietary formats that can be +read and edited only by proprietary word processors, @acronym{SGML} or +@acronym{XML} for which the @acronym{DTD} and/or processing tools are +not generally available, and the machine-generated @acronym{HTML}, +PostScript or @acronym{PDF} produced by some word processors for +output purposes only. + +The ``Title Page'' means, for a printed book, the title page itself, +plus such following pages as are needed to hold, legibly, the material +this License requires to appear in the title page. For works in +formats which do not have any title page as such, ``Title Page'' means +the text near the most prominent appearance of the work's title, +preceding the beginning of the body of the text. + +A section ``Entitled XYZ'' means a named subunit of the Document whose +title either is precisely XYZ or contains XYZ in parentheses following +text that translates XYZ in another language. (Here XYZ stands for a +specific section name mentioned below, such as ``Acknowledgements'', +``Dedications'', ``Endorsements'', or ``History''.) To ``Preserve the Title'' +of such a section when you modify the Document means that it remains a +section ``Entitled XYZ'' according to this definition. + +The Document may include Warranty Disclaimers next to the notice which +states that this License applies to the Document. These Warranty +Disclaimers are considered to be included by reference in this +License, but only as regards disclaiming warranties: any other +implication that these Warranty Disclaimers may have is void and has +no effect on the meaning of this License. + +@item +VERBATIM COPYING + +You may copy and distribute the Document in any medium, either +commercially or noncommercially, provided that this License, the +copyright notices, and the license notice saying this License applies +to the Document are reproduced in all copies, and that you add no other +conditions whatsoever to those of this License. You may not use +technical measures to obstruct or control the reading or further +copying of the copies you make or distribute. However, you may accept +compensation in exchange for copies. If you distribute a large enough +number of copies you must also follow the conditions in section 3. + +You may also lend copies, under the same conditions stated above, and +you may publicly display copies. + +@item +COPYING IN QUANTITY + +If you publish printed copies (or copies in media that commonly have +printed covers) of the Document, numbering more than 100, and the +Document's license notice requires Cover Texts, you must enclose the +copies in covers that carry, clearly and legibly, all these Cover +Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on +the back cover. Both covers must also clearly and legibly identify +you as the publisher of these copies. The front cover must present +the full title with all words of the title equally prominent and +visible. You may add other material on the covers in addition. +Copying with changes limited to the covers, as long as they preserve +the title of the Document and satisfy these conditions, can be treated +as verbatim copying in other respects. + +If the required texts for either cover are too voluminous to fit +legibly, you should put the first ones listed (as many as fit +reasonably) on the actual cover, and continue the rest onto adjacent +pages. + +If you publish or distribute Opaque copies of the Document numbering +more than 100, you must either include a machine-readable Transparent +copy along with each Opaque copy, or state in or with each Opaque copy +a computer-network location from which the general network-using +public has access to download using public-standard network protocols +a complete Transparent copy of the Document, free of added material. +If you use the latter option, you must take reasonably prudent steps, +when you begin distribution of Opaque copies in quantity, to ensure +that this Transparent copy will remain thus accessible at the stated +location until at least one year after the last time you distribute an +Opaque copy (directly or through your agents or retailers) of that +edition to the public. + +It is requested, but not required, that you contact the authors of the +Document well before redistributing any large number of copies, to give +them a chance to provide you with an updated version of the Document. + +@item +MODIFICATIONS + +You may copy and distribute a Modified Version of the Document under +the conditions of sections 2 and 3 above, provided that you release +the Modified Version under precisely this License, with the Modified +Version filling the role of the Document, thus licensing distribution +and modification of the Modified Version to whoever possesses a copy +of it. In addition, you must do these things in the Modified Version: + +@enumerate A +@item +Use in the Title Page (and on the covers, if any) a title distinct +from that of the Document, and from those of previous versions +(which should, if there were any, be listed in the History section +of the Document). You may use the same title as a previous version +if the original publisher of that version gives permission. + +@item +List on the Title Page, as authors, one or more persons or entities +responsible for authorship of the modifications in the Modified +Version, together with at least five of the principal authors of the +Document (all of its principal authors, if it has fewer than five), +unless they release you from this requirement. + +@item +State on the Title page the name of the publisher of the +Modified Version, as the publisher. + +@item +Preserve all the copyright notices of the Document. + +@item +Add an appropriate copyright notice for your modifications +adjacent to the other copyright notices. + +@item +Include, immediately after the copyright notices, a license notice +giving the public permission to use the Modified Version under the +terms of this License, in the form shown in the Addendum below. + +@item +Preserve in that license notice the full lists of Invariant Sections +and required Cover Texts given in the Document's license notice. + +@item +Include an unaltered copy of this License. + +@item +Preserve the section Entitled ``History'', Preserve its Title, and add +to it an item stating at least the title, year, new authors, and +publisher of the Modified Version as given on the Title Page. If +there is no section Entitled ``History'' in the Document, create one +stating the title, year, authors, and publisher of the Document as +given on its Title Page, then add an item describing the Modified +Version as stated in the previous sentence. + +@item +Preserve the network location, if any, given in the Document for +public access to a Transparent copy of the Document, and likewise +the network locations given in the Document for previous versions +it was based on. These may be placed in the ``History'' section. +You may omit a network location for a work that was published at +least four years before the Document itself, or if the original +publisher of the version it refers to gives permission. + +@item +For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve +the Title of the section, and preserve in the section all the +substance and tone of each of the contributor acknowledgements and/or +dedications given therein. + +@item +Preserve all the Invariant Sections of the Document, +unaltered in their text and in their titles. Section numbers +or the equivalent are not considered part of the section titles. + +@item +Delete any section Entitled ``Endorsements''. Such a section +may not be included in the Modified Version. + +@item +Do not retitle any existing section to be Entitled ``Endorsements'' or +to conflict in title with any Invariant Section. + +@item +Preserve any Warranty Disclaimers. +@end enumerate + +If the Modified Version includes new front-matter sections or +appendices that qualify as Secondary Sections and contain no material +copied from the Document, you may at your option designate some or all +of these sections as invariant. To do this, add their titles to the +list of Invariant Sections in the Modified Version's license notice. +These titles must be distinct from any other section titles. + +You may add a section Entitled ``Endorsements'', provided it contains +nothing but endorsements of your Modified Version by various +parties---for example, statements of peer review or that the text has +been approved by an organization as the authoritative definition of a +standard. + +You may add a passage of up to five words as a Front-Cover Text, and a +passage of up to 25 words as a Back-Cover Text, to the end of the list +of Cover Texts in the Modified Version. Only one passage of +Front-Cover Text and one of Back-Cover Text may be added by (or +through arrangements made by) any one entity. If the Document already +includes a cover text for the same cover, previously added by you or +by arrangement made by the same entity you are acting on behalf of, +you may not add another; but you may replace the old one, on explicit +permission from the previous publisher that added the old one. + +The author(s) and publisher(s) of the Document do not by this License +give permission to use their names for publicity for or to assert or +imply endorsement of any Modified Version. + +@item +COMBINING DOCUMENTS + +You may combine the Document with other documents released under this +License, under the terms defined in section 4 above for modified +versions, provided that you include in the combination all of the +Invariant Sections of all of the original documents, unmodified, and +list them all as Invariant Sections of your combined work in its +license notice, and that you preserve all their Warranty Disclaimers. + +The combined work need only contain one copy of this License, and +multiple identical Invariant Sections may be replaced with a single +copy. If there are multiple Invariant Sections with the same name but +different contents, make the title of each such section unique by +adding at the end of it, in parentheses, the name of the original +author or publisher of that section if known, or else a unique number. +Make the same adjustment to the section titles in the list of +Invariant Sections in the license notice of the combined work. + +In the combination, you must combine any sections Entitled ``History'' +in the various original documents, forming one section Entitled +``History''; likewise combine any sections Entitled ``Acknowledgements'', +and any sections Entitled ``Dedications''. You must delete all +sections Entitled ``Endorsements.'' + +@item +COLLECTIONS OF DOCUMENTS + +You may make a collection consisting of the Document and other documents +released under this License, and replace the individual copies of this +License in the various documents with a single copy that is included in +the collection, provided that you follow the rules of this License for +verbatim copying of each of the documents in all other respects. + +You may extract a single document from such a collection, and distribute +it individually under this License, provided you insert a copy of this +License into the extracted document, and follow this License in all +other respects regarding verbatim copying of that document. + +@item +AGGREGATION WITH INDEPENDENT WORKS + +A compilation of the Document or its derivatives with other separate +and independent documents or works, in or on a volume of a storage or +distribution medium, is called an ``aggregate'' if the copyright +resulting from the compilation is not used to limit the legal rights +of the compilation's users beyond what the individual works permit. +When the Document is included in an aggregate, this License does not +apply to the other works in the aggregate which are not themselves +derivative works of the Document. + +If the Cover Text requirement of section 3 is applicable to these +copies of the Document, then if the Document is less than one half of +the entire aggregate, the Document's Cover Texts may be placed on +covers that bracket the Document within the aggregate, or the +electronic equivalent of covers if the Document is in electronic form. +Otherwise they must appear on printed covers that bracket the whole +aggregate. + +@item +TRANSLATION + +Translation is considered a kind of modification, so you may +distribute translations of the Document under the terms of section 4. +Replacing Invariant Sections with translations requires special +permission from their copyright holders, but you may include +translations of some or all Invariant Sections in addition to the +original versions of these Invariant Sections. You may include a +translation of this License, and all the license notices in the +Document, and any Warranty Disclaimers, provided that you also include +the original English version of this License and the original versions +of those notices and disclaimers. In case of a disagreement between +the translation and the original version of this License or a notice +or disclaimer, the original version will prevail. + +If a section in the Document is Entitled ``Acknowledgements'', +``Dedications'', or ``History'', the requirement (section 4) to Preserve +its Title (section 1) will typically require changing the actual +title. + +@item +TERMINATION + +You may not copy, modify, sublicense, or distribute the Document except +as expressly provided for under this License. Any other attempt to +copy, modify, sublicense or distribute the Document is void, and will +automatically terminate your rights under this License. However, +parties who have received copies, or rights, from you under this +License will not have their licenses terminated so long as such +parties remain in full compliance. + +@item +FUTURE REVISIONS OF THIS LICENSE + +The Free Software Foundation may publish new, revised versions +of the GNU Free Documentation License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. See +@uref{http://www.gnu.org/copyleft/}. + +Each version of the License is given a distinguishing version number. +If the Document specifies that a particular numbered version of this +License ``or any later version'' applies to it, you have the option of +following the terms and conditions either of that specified version or +of any later version that has been published (not as a draft) by the +Free Software Foundation. If the Document does not specify a version +number of this License, you may choose any version ever published (not +as a draft) by the Free Software Foundation. +@end enumerate + +@page +@heading ADDENDUM: How to use this License for your documents + +To use this License in a document you have written, include a copy of +the License in the document and put the following copyright and +license notices just after the title page: + +@smallexample +@group + Copyright (C) @var{year} @var{your name}. + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU Free Documentation License, Version 1.2 + or any later version published by the Free Software Foundation; + with no Invariant Sections, no Front-Cover Texts, and no Back-Cover + Texts. A copy of the license is included in the section entitled ``GNU + Free Documentation License''. +@end group +@end smallexample + +If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts, +replace the ``with@dots{}Texts.'' line with this: + +@smallexample +@group + with the Invariant Sections being @var{list their titles}, with + the Front-Cover Texts being @var{list}, and with the Back-Cover Texts + being @var{list}. +@end group +@end smallexample + +If you have Invariant Sections without Cover Texts, or some other +combination of the three, merge those two alternatives to suit the +situation. + +If your document contains nontrivial examples of program code, we +recommend releasing these examples in parallel under your choice of +free software license, such as the GNU General Public License, +to permit their use in free software. + +@c Local Variables: +@c ispell-local-pdict: "ispell-dict" +@c End: + diff --git a/guile/meson.build b/guile/meson.build new file mode 100644 index 0000000..aceada3 --- /dev/null +++ b/guile/meson.build @@ -0,0 +1,114 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# create a shell script for compiling from the source dirs +compile_scm_conf = configuration_data() +compile_scm_conf.set('abs_builddir', meson.current_build_dir()) +compile_scm_conf.set('guild', 'guild') +compile_scm=configure_file( + input: 'compile-scm.in', + output: 'compile-scm', + configuration: compile_scm_conf, + install: false +) +run_command('chmod', '+x', compile_scm, check: true) +scm_compiler=join_paths(meson.current_build_dir(), 'compile-scm') + +# +# NOTE: snarfing works but you get: +# ,---- +# | cc1plus: warning: command-line option ‘-std=gnu11’ is valid for C/ObjC +# | but not for C++ +# `---- +# this is because the snarf-script hardcodes the '-std=gnu11' but we're +# building for c++; even worse, e.g. on some MacOS, the warning is a +# hard error. +# +# We can override flag through a env variable CPP; but then we _also_ need to +# override the compiler, so e.g. CPP="g++ -std=c++17'; but it's a bit +# hairy/ugly/fragile to derive the raw compiler name in meson; also the +# generator expression doesn't take an 'env:' parameter, so we'd need +# to rewrite using custom_target... +# +# for now, we avoid all that by simply including the generated files. +do_snarf=false + +if do_snarf + snarf = find_program('guile-snarf3.0','guile-snarf') + # there must be a better way of feeding the include paths to snarf... + snarf_args=['-o', '@OUTPUT@', '@INPUT@', '-I' + meson.current_source_dir() + '/..', + '-I' + meson.current_source_dir() + '/../lib', + '-I' + meson.current_build_dir() + '/..'] + snarf_args += '-I' + join_paths(glib_dep.get_pkgconfig_variable('includedir'), + 'glib-2.0') + snarf_args += '-I' + join_paths(glib_dep.get_pkgconfig_variable('libdir'), + 'glib-2.0', 'include') + snarf_args += '-I' + join_paths(guile_dep.get_pkgconfig_variable('includedir'), + 'guile', '3.0') + snarf_gen=generator(snarf, + output: '@BASENAME@.x', + arguments: snarf_args) + snarf_srcs=['mu-guile.cc', 'mu-guile-message.cc'] + snarf_x=snarf_gen.process(snarf_srcs) +else + snarf_x = [ 'mu-guile-message.x', 'mu-guile.x' ] +endif + +lib_guile_mu = shared_module( + 'guile-mu', + [ 'mu-guile.cc', + 'mu-guile-message.cc' ], + dependencies: [guile_dep, glib_dep, lib_mu_dep, config_h_dep, thread_dep ], + install: true, + install_dir: guile_extension_dir +) + +if makeinfo.found() + custom_target('mu_guile_info', + input: 'mu-guile.texi', + output: 'mu-guile.info', + install: true, + install_dir: infodir, + command: [makeinfo, + '-o', join_paths(meson.current_build_dir(), 'mu-guile.info'), + join_paths(meson.current_source_dir(), 'mu-guile.texi'), + '-I', join_paths(meson.current_build_dir(), '..')]) + if install_info.found() + infodir = join_paths(get_option('prefix') / get_option('infodir')) + meson.add_install_script(install_info_script, infodir, 'mu-guile.info') + endif +endif + +guile_scm_dir=join_paths(datadir, 'guile', 'site', '3.0') +install_data(['mu.scm'], install_dir: guile_scm_dir) +guile_scm_mu_dir=join_paths(guile_scm_dir, 'mu') +foreach mod : ['script.scm', 'message.scm', 'stats.scm', 'plot.scm'] + install_data(join_paths('mu', mod), install_dir: guile_scm_mu_dir) +endforeach + +mu_guile_scripts=[ + join_paths('scripts', 'find-dups.scm'), + join_paths('scripts', 'msgs-count.scm'), + join_paths('scripts', 'histogram.scm')] +mu_guile_script_dir=join_paths(datadir, 'mu', 'scripts') +install_data(mu_guile_scripts, install_dir: mu_guile_script_dir) + +guile_builddir=meson.current_build_dir() + +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/guile/mu-guile-message.cc b/guile/mu-guile-message.cc new file mode 100644 index 0000000..281ed7c --- /dev/null +++ b/guile/mu-guile-message.cc @@ -0,0 +1,485 @@ +/* +** Copyright (C) 2011-2023 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include +#include "mu-guile-message.hh" + +#include "message/mu-message.hh" +#include "utils/mu-utils.hh" + +#include +#include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wredundant-decls" +#include +#pragma GCC diagnostic pop + +#include "mu-guile.hh" + +#include +#include + +using namespace Mu; + +/* pseudo field, not in Xapian */ +constexpr auto MU_GUILE_MSG_FIELD_ID_TIMESTAMP = Field::id_size() + 1; + +/* some symbols */ +static SCM SYMB_PRIO_LOW, SYMB_PRIO_NORMAL, SYMB_PRIO_HIGH; +static std::array SYMB_FLAGS; +static SCM SYMB_CONTACT_TO, SYMB_CONTACT_CC, SYMB_CONTACT_BCC, SYMB_CONTACT_FROM; +static long MSG_TAG; + + +using MessageSPtr = std::unique_ptr; + +static gboolean +mu_guile_scm_is_msg(SCM scm) +{ + return SCM_NIMP(scm) && (long)SCM_CAR(scm) == MSG_TAG; +} + +static SCM +message_scm_create(Xapian::Document&& doc) +{ + /* placement-new */ + + void *scm_mem{scm_gc_malloc(sizeof(Message), "msg")}; + Message* msgp = new(scm_mem)Message(std::move(doc)); + + SCM_RETURN_NEWSMOB(MSG_TAG, msgp); +} + +static const Message* +message_from_scm(SCM msg_smob) +{ + return reinterpret_cast(SCM_CDR(msg_smob)); +} + +static size_t +message_scm_free(SCM msg_smob) +{ + if (auto msg = message_from_scm(msg_smob); msg) + msg->~Message(); + + return sizeof(Message); +} + +static int +message_scm_print(SCM msg_smob, SCM port, scm_print_state* pstate) +{ + scm_puts("#path().c_str(), port); + + scm_puts(">", port); + return 1; +} + +struct FlagData { + Flags flags; + SCM lst; +}; + +#define MU_GUILE_INITIALIZED_OR_ERROR \ + do { \ + if (!(mu_guile_initialized())) { \ + mu_guile_error(FUNC_NAME, \ + 0, \ + "mu not initialized; call mu:initialize", \ + SCM_UNDEFINED); \ + return SCM_UNSPECIFIED; \ + } \ + } while (0) + + +static SCM +get_flags_scm(const Message& msg) +{ + SCM lst{SCM_EOL}; + const auto flags{msg.flags()}; + + for (auto i = 0; i != AllMessageFlagInfos.size(); ++i) { + const auto& info{AllMessageFlagInfos.at(i)}; + if (any_of(info.flag & flags)) + scm_append_x(scm_list_2(lst, scm_list_1(SYMB_FLAGS.at(i)))); + } + + return lst; +} + +static SCM +get_prio_scm(const Message& msg) +{ + switch (msg.priority()) { + case Priority::Low: return SYMB_PRIO_LOW; + case Priority::Normal: return SYMB_PRIO_NORMAL; + case Priority::High: return SYMB_PRIO_HIGH; + + default: g_return_val_if_reached(SCM_UNDEFINED); + } +} + +static SCM +msg_string_list_field(const Message& msg, Field::Id field_id) +{ + SCM scmlst{SCM_EOL}; + for (auto&& val: msg.document().string_vec_value(field_id)) { + SCM item; + item = scm_list_1(mu_guile_scm_from_string(val)); + scmlst = scm_append_x(scm_list_2(scmlst, item)); + } + + return scmlst; +} + +static SCM +msg_contact_list_field(const Message& msg, Field::Id field_id) +{ + return scm_from_utf8_string( + to_string(msg.document().contacts_value(field_id)).c_str()); +} + +static SCM +get_body(const Message& msg, bool html) +{ + if (const auto body = html ? msg.body_html() : msg.body_text(); body) + return mu_guile_scm_from_string(*body); + else + return SCM_BOOL_F; +} + +SCM_DEFINE(get_field, + "mu:c:get-field", + 2, + 0, + 0, + (SCM MSG, SCM FIELD), + "Get the field FIELD from message MSG.\n") +#define FUNC_NAME s_get_field +{ + SCM_ASSERT(mu_guile_scm_is_msg(MSG), MSG, SCM_ARG1, FUNC_NAME); + auto msg{message_from_scm(MSG)}; + SCM_ASSERT(msg, MSG, SCM_ARG1, FUNC_NAME); + + SCM_ASSERT(scm_integer_p(FIELD), FIELD, SCM_ARG2, FUNC_NAME); + const auto field_opt{field_from_number(static_cast(scm_to_int(FIELD)))}; + SCM_ASSERT(!!field_opt, FIELD, SCM_ARG2, FUNC_NAME); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch (field_opt->id) { + case Field::Id::Priority: + return get_prio_scm(*msg); + case Field::Id::Flags: + return get_flags_scm(*msg); + case Field::Id::BodyText: + return get_body(*msg, false); + default: + break; + } +#pragma GCC diagnostic pop + + switch (field_opt->type) { + case Field::Type::String: + return mu_guile_scm_from_string(msg->document().string_value(field_opt->id)); + case Field::Type::ByteSize: + case Field::Type::TimeT: + case Field::Type::Integer: + return scm_from_uint(msg->document().integer_value(field_opt->id)); + case Field::Type::StringList: + return msg_string_list_field(*msg, field_opt->id); + case Field::Type::ContactList: + return msg_contact_list_field(*msg, field_opt->id); + default: + SCM_ASSERT(0, FIELD, SCM_ARG2, FUNC_NAME); + } + + return SCM_UNSPECIFIED; +} +#undef FUNC_NAME + +static SCM +contacts_to_list(const Message& msg, Option field_id) +{ + SCM list{SCM_EOL}; + + const auto contacts{field_id ? + msg.document().contacts_value(*field_id) : + msg.all_contacts()}; + + for (auto&& contact: contacts) { + SCM item{scm_list_1( + scm_cons(mu_guile_scm_from_string(contact.name), + mu_guile_scm_from_string(contact.email)))}; + list = scm_append_x(scm_list_2(list, item)); + } + + return list; +} + +SCM_DEFINE(get_contacts, + "mu:c:get-contacts", + 2, + 0, + 0, + (SCM MSG, SCM CONTACT_TYPE), + "Get a list of contact information pairs.\n") +#define FUNC_NAME s_get_contacts +{ + SCM list; + + MU_GUILE_INITIALIZED_OR_ERROR; + + SCM_ASSERT(mu_guile_scm_is_msg(MSG), MSG, SCM_ARG1, FUNC_NAME); + auto msg{message_from_scm(MSG)}; + SCM_ASSERT(msg, MSG, SCM_ARG1, FUNC_NAME); + + SCM_ASSERT(scm_symbol_p(CONTACT_TYPE) || scm_is_bool(CONTACT_TYPE), + CONTACT_TYPE, + SCM_ARG2, + FUNC_NAME); + + if (CONTACT_TYPE == SCM_BOOL_F) + return SCM_UNSPECIFIED; /* nothing to do */ + + Option field_id; + if (CONTACT_TYPE == SCM_BOOL_T) + field_id = {}; /* get all */ + else { + if (scm_is_eq(CONTACT_TYPE, SYMB_CONTACT_TO)) + field_id = Field::Id::To; + else if (scm_is_eq(CONTACT_TYPE, SYMB_CONTACT_CC)) + field_id = Field::Id::Cc; + else if (scm_is_eq(CONTACT_TYPE, SYMB_CONTACT_BCC)) + field_id = Field::Id::Bcc; + else if (scm_is_eq(CONTACT_TYPE, SYMB_CONTACT_FROM)) + field_id = Field::Id::From; + else { + mu_guile_error(FUNC_NAME, 0, "invalid contact type", SCM_UNDEFINED); + return SCM_UNSPECIFIED; + } + } + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-function-type" + list = contacts_to_list(*msg, field_id); +#pragma GCC diagnostic pop + + /* explicitly close the file backend, so we won't run out of fds */ + + + return list; +} +#undef FUNC_NAME + +SCM_DEFINE(get_parts, + "mu:c:get-parts", + 1, + 1, + 0, + (SCM MSG, SCM ATTS_ONLY), + "Get the list of mime-parts for MSG. If ATTS_ONLY is #t, only" + "get parts that are (look like) attachments. The resulting list has " + "elements which are list of the form (index name mime-type size).\n") +#define FUNC_NAME s_get_parts +{ + MU_GUILE_INITIALIZED_OR_ERROR; + + SCM_ASSERT(mu_guile_scm_is_msg(MSG), MSG, SCM_ARG1, FUNC_NAME); + auto msg{message_from_scm(MSG)}; + SCM_ASSERT(msg, MSG, SCM_ARG1, FUNC_NAME); + SCM_ASSERT(scm_is_bool(ATTS_ONLY), ATTS_ONLY, SCM_ARG2, FUNC_NAME); + + SCM attlist = SCM_EOL; /* empty list */ + bool attachments_only = ATTS_ONLY == SCM_BOOL_T ? TRUE : FALSE; + + size_t n{}; + for (auto&& part: msg->parts()) { + + if (attachments_only && !part.is_attachment()) + continue; + + const auto mime_type{part.mime_type()}; + const auto filename{part.cooked_filename()}; + + SCM elm = scm_list_5( + /* msg */ + mu_guile_scm_from_string(msg->path().c_str()), + /* index */ + scm_from_uint(n++), + /* filename or #f */ + filename ? mu_guile_scm_from_string(*filename) : SCM_BOOL_F, + /* mime-type */ + mime_type ? mu_guile_scm_from_string(*mime_type) : SCM_BOOL_F, + /* size */ + part.size() > 0 ? scm_from_uint(part.size()) : SCM_BOOL_F); + + attlist = scm_cons(elm, attlist); + } + + /* explicitly close the file backend, so we won't run of fds */ + msg->unload_mime_message(); + + return attlist; +} +#undef FUNC_NAME + +SCM_DEFINE(get_header, + "mu:c:get-header", + 2, + 0, + 0, + (SCM MSG, SCM HEADER), + "Get an arbitrary HEADER from MSG.\n") +#define FUNC_NAME s_get_header +{ + MU_GUILE_INITIALIZED_OR_ERROR; + + SCM_ASSERT(mu_guile_scm_is_msg(MSG), MSG, SCM_ARG1, FUNC_NAME); + auto msg{message_from_scm(MSG)}; + SCM_ASSERT(msg, MSG, SCM_ARG1, FUNC_NAME); + + SCM_ASSERT(scm_is_string(HEADER) || HEADER == SCM_UNDEFINED, HEADER, SCM_ARG2, FUNC_NAME); + + char *header = scm_to_utf8_string(HEADER); + SCM val = mu_guile_scm_from_string(msg->header(header).value_or("")); + free(header); + + /* explicitly close the file backend, so we won't run of fds */ + msg->unload_mime_message(); + + return val; +} +#undef FUNC_NAME +SCM_DEFINE(for_each_message, + "mu:c:for-each-message", + 3, + 0, + 0, + (SCM FUNC, SCM EXPR, SCM MAXNUM), + "Call FUNC for each msg in the message store matching EXPR. EXPR is" + "either a string containing a mu search expression or a boolean; in the former " + "case, limit the messages to only those matching the expression, in the " + "latter case, match /all/ messages if the EXPR equals #t, and match " + "none if EXPR equals #f.") +#define FUNC_NAME s_for_each_message +{ + char* expr{}; + + MU_GUILE_INITIALIZED_OR_ERROR; + + SCM_ASSERT(scm_procedure_p(FUNC), FUNC, SCM_ARG1, FUNC_NAME); + SCM_ASSERT(scm_is_bool(EXPR) || scm_is_string(EXPR), EXPR, SCM_ARG2, FUNC_NAME); + SCM_ASSERT(scm_is_integer(MAXNUM), MAXNUM, SCM_ARG3, FUNC_NAME); + + if (EXPR == SCM_BOOL_F) + return SCM_UNSPECIFIED; /* nothing to do */ + + if (EXPR == SCM_BOOL_T) + expr = strdup("\"\""); /* note, "" matches *all* messages */ + else + expr = scm_to_utf8_string(EXPR); + + const auto res = mu_guile_store().run_query(expr,{}, {}, scm_to_int(MAXNUM)); + free(expr); + if (!res) + return SCM_UNSPECIFIED; + + for (auto&& mi : *res) { + if (auto xdoc{mi.document()}; xdoc) { + scm_call_1(FUNC, message_scm_create(std::move(xdoc.value()))); + } + } + + return SCM_UNSPECIFIED; +} +#undef FUNC_NAME + +static SCM +register_symbol(const char* name) +{ + SCM scm; + + scm = scm_from_utf8_symbol(name); + scm_c_define(name, scm); + scm_c_export(name, NULL); + + return scm; +} + +static void +define_symbols(void) +{ + SYMB_CONTACT_TO = register_symbol("mu:contact:to"); + SYMB_CONTACT_CC = register_symbol("mu:contact:cc"); + SYMB_CONTACT_FROM = register_symbol("mu:contact:from"); + SYMB_CONTACT_BCC = register_symbol("mu:contact:bcc"); + + SYMB_PRIO_LOW = register_symbol("mu:prio:low"); + SYMB_PRIO_NORMAL = register_symbol("mu:prio:normal"); + SYMB_PRIO_HIGH = register_symbol("mu:prio:high"); + + for (auto i = 0U; i != AllMessageFlagInfos.size(); ++i) { + const auto& info{AllMessageFlagInfos.at(i)}; + const auto name = "mu:flag:" + std::string{info.name}; + SYMB_FLAGS[i] = register_symbol(name.c_str()); + } +} +static void +define_vars(void) +{ + field_for_each([](auto&& field){ + + auto defvar = [&](auto&& fname, auto&& ffield) { + const auto name{"mu:field:" + std::string{fname}}; + scm_c_define(name.c_str(), scm_from_uint(field.value_no())); + scm_c_export(name.c_str(), NULL); + }; + + // define for both name and (if exists) alias. + if (!field.name.empty()) + defvar(field.name, field); + if (!field.alias.empty()) + defvar(field.alias, field); + }); + + /* non-Xapian field: timestamp */ + scm_c_define("mu:field:timestamp", + scm_from_uint(MU_GUILE_MSG_FIELD_ID_TIMESTAMP)); + scm_c_export("mu:field:timestamp", NULL); + +} + +void* +mu_guile_message_init(void* data) +{ + MSG_TAG = scm_make_smob_type("message", sizeof(Message)); + + scm_set_smob_free(MSG_TAG, message_scm_free); + scm_set_smob_print(MSG_TAG, message_scm_print); + + define_vars(); + define_symbols(); + +#ifndef SCM_MAGIC_SNARFER +#include "mu-guile-message.x" +#endif /*SCM_MAGIC_SNARFER*/ + + return NULL; +} diff --git a/guile/mu-guile-message.hh b/guile/mu-guile-message.hh new file mode 100644 index 0000000..0e7201d --- /dev/null +++ b/guile/mu-guile-message.hh @@ -0,0 +1,34 @@ +/* +** Copyright (C) 2011-2020 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_GUILE_MESSAGE_H__ +#define MU_GUILE_MESSAGE_H__ + +/** + * Initialize this mu guile module. + * + * @param data +q * + * @return + */ +extern "C" { +void* mu_guile_message_init(void* data); +} + +#endif /*MU_GUILE_MESSAGE_HH__*/ diff --git a/guile/mu-guile-message.x b/guile/mu-guile-message.x new file mode 100644 index 0000000..6127b39 --- /dev/null +++ b/guile/mu-guile-message.x @@ -0,0 +1,6 @@ +/* cpp arguments: mu-guile-message.cc -DHAVE_CONFIG_H -I. -I.. -I../lib -I/usr/local/include/guile/3.0 -pthread -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -I/usr/include/sysprof-4 -I/usr/include/libmount -I/usr/include/blkid -pthread -fno-strict-aliasing -Wall -Wextra -Wundef -Wwrite-strings -Wpointer-arith -Wmissing-declarations -Wredundant-decls -Wno-unused-parameter -Wno-missing-field-initializers -Wformat=2 -Wcast-align -Wformat-nonliteral -Wformat-security -Wsign-compare -Wstrict-aliasing -Wshadow -Winline -Wpacked -Wmissing-format-attribute -Wmissing-noreturn -Winit-self -Wmissing-include-dirs -Wunused-but-set-variable -Warray-bounds -Wreturn-type -Wno-overloaded-virtual -Wswitch-enum -Wswitch-default -Wno-error=unused-parameter -Wno-error=missing-field-initializers -Wno-error=overloaded-virtual -Wno-redundant-decls -Wno-missing-declarations -Wno-suggest-attribute=noreturn -O2 -Wno-inline */ +scm_c_define_gsubr (s_get_field, 2, 0, 0, (scm_t_subr) get_field);; +scm_c_define_gsubr (s_get_contacts, 2, 0, 0, (scm_t_subr) get_contacts);; +scm_c_define_gsubr (s_get_parts, 1, 1, 0, (scm_t_subr) get_parts);; +scm_c_define_gsubr (s_get_header, 2, 0, 0, (scm_t_subr) get_header);; +scm_c_define_gsubr (s_for_each_message, 3, 0, 0, (scm_t_subr) for_each_message);; diff --git a/guile/mu-guile.cc b/guile/mu-guile.cc new file mode 100644 index 0000000..44659aa --- /dev/null +++ b/guile/mu-guile.cc @@ -0,0 +1,250 @@ +/* +** Copyright (C) 2011-2023 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include + +#include "mu-guile.hh" + +#include +#include + +#include +#include + +#include + +using namespace Mu; + +SCM +mu_guile_scm_from_string(const std::string& str) +{ + if (str.empty()) + return SCM_BOOL_F; + else + return scm_from_stringn(str.c_str(), str.size(), + "UTF-8", + SCM_FAILED_CONVERSION_QUESTION_MARK); +} + +SCM +mu_guile_error(const char* func_name, int status, const char* fmt, SCM args) +{ + scm_error_scm(scm_from_locale_symbol("MuError"), + scm_from_utf8_string(func_name ? func_name : ""), + scm_from_utf8_string(fmt), + args, + scm_list_1(scm_from_int(status))); + + return SCM_UNSPECIFIED; +} + +SCM +mu_guile_g_error(const char* func_name, GError* err) +{ + scm_error_scm(scm_from_locale_symbol("MuError"), + scm_from_utf8_string(func_name), + scm_from_utf8_string(err ? err->message : "error"), + SCM_UNDEFINED, + SCM_UNDEFINED); + + return SCM_UNSPECIFIED; +} + +/* there can be only one */ + +static Option StoreSingleton = Nothing; + +static bool +mu_guile_init_instance(const std::string& muhome) try { + setlocale(LC_ALL, ""); + + const auto path{runtime_path(RuntimePath::XapianDb, muhome)}; + auto store = Store::make(path); + if (!store) { + mu_critical("error creating store @ %s: %s", path, + store.error().what()); + throw store.error(); + } else + StoreSingleton.emplace(std::move(store.value())); + + mu_debug("mu-guile: opened store @ {} (n={}); maildir: {}", + StoreSingleton->path(), + StoreSingleton->size(), + StoreSingleton->root_maildir()); + + return true; + +} catch (const Xapian::Error& xerr) { + mu_critical("{}: xapian error '{}'", __func__, xerr.get_msg()); + return false; +} catch (const std::runtime_error& re) { + mu_critical("{}: error: {}", __func__, re.what()); + return false; +} catch (const std::exception& e) { + mu_critical("{}: caught exception: {}", __func__, e.what()); + return false; +} catch (...) { + mu_critical("{}: caught exception", __func__); + return false; +} + +static void +mu_guile_uninit_instance() +{ + StoreSingleton.reset(); +} + +Mu::Store& +mu_guile_store() +{ + if (!StoreSingleton) + mu_error("mu guile not initialized"); + + return StoreSingleton.value(); +} + +gboolean +mu_guile_initialized() +{ + g_debug("initialized ? %u", !!StoreSingleton); + + return !!StoreSingleton; +} + +SCM_DEFINE_PUBLIC(mu_initialize, + "mu:initialize", + 0, + 1, + 0, + (SCM MUHOME), + "Initialize mu - needed before you call any of the other " + "functions. Optionally, you can provide MUHOME which should be an " + "absolute path to your mu home directory " + "-- typically, the default, ~/.cache/mu, should be just fine.") +#define FUNC_NAME s_mu_initialize +{ + char* muhome; + + SCM_ASSERT(scm_is_string(MUHOME) || MUHOME == SCM_BOOL_F || SCM_UNBNDP(MUHOME), + MUHOME, + SCM_ARG1, + FUNC_NAME); + + if (mu_guile_initialized()) + return mu_guile_error(FUNC_NAME, 0, "Already initialized", SCM_UNSPECIFIED); + + if (SCM_UNBNDP(MUHOME) || MUHOME == SCM_BOOL_F) + muhome = NULL; + else + muhome = scm_to_utf8_string(MUHOME); + + if (!mu_guile_init_instance(muhome ? muhome : "")) { + free(muhome); + mu_guile_error(FUNC_NAME, 0, "Failed to initialize mu", SCM_UNSPECIFIED); + return SCM_UNSPECIFIED; + } + + g_debug("mu-guile: initialized @ %s", muhome ? muhome : ""); + free(muhome); + + /* cleanup when we're exiting */ + atexit(mu_guile_uninit_instance); + + return SCM_UNSPECIFIED; +} +#undef FUNC_NAME + +SCM_DEFINE_PUBLIC(mu_initialized_p, + "mu:initialized?", + 0, + 0, + 0, + (void), + "Whether mu is initialized or not.\n") +#define FUNC_NAME s_mu_initialized_p +{ + return mu_guile_initialized() ? SCM_BOOL_T : SCM_BOOL_F; +} +#undef FUNC_NAME + +SCM_DEFINE(log_func, + "mu:c:log", + 1, + 0, + 1, + (SCM LEVEL, SCM FRM, SCM ARGS), + "log some message at LEVEL using a list of ARGS applied to FRM" + "(in 'simple-format' notation).\n") +#define FUNC_NAME s_log_func +{ + gchar* output; + SCM str; + int level; + + SCM_ASSERT(scm_integer_p(LEVEL), LEVEL, SCM_ARG1, FUNC_NAME); + SCM_ASSERT(scm_is_string(FRM), FRM, SCM_ARG2, ""); + SCM_VALIDATE_REST_ARGUMENT(ARGS); + + level = scm_to_int(LEVEL); + if (level != G_LOG_LEVEL_MESSAGE && level != G_LOG_LEVEL_WARNING && + level != G_LOG_LEVEL_CRITICAL) + return mu_guile_error(FUNC_NAME, 0, "invalid log level", SCM_UNSPECIFIED); + + str = scm_simple_format(SCM_BOOL_F, FRM, ARGS); + + if (!scm_is_string(str)) + return SCM_UNSPECIFIED; + + output = scm_to_utf8_string(str); + g_log(G_LOG_DOMAIN, (GLogLevelFlags)level, "%s", output); + free(output); + + return SCM_UNSPECIFIED; +} +#undef FUNC_NAME + +static struct { + const char* name; + unsigned val; +} VAR_PAIRS[] = { + + {"mu:message", G_LOG_LEVEL_MESSAGE}, + {"mu:warning", G_LOG_LEVEL_WARNING}, + {"mu:critical", G_LOG_LEVEL_CRITICAL}}; + +static void +define_vars(void) +{ + unsigned u; + for (u = 0; u != G_N_ELEMENTS(VAR_PAIRS); ++u) { + scm_c_define(VAR_PAIRS[u].name, scm_from_uint(VAR_PAIRS[u].val)); + scm_c_export(VAR_PAIRS[u].name, NULL); + } +} + +void* +mu_guile_init(void* data) +{ + define_vars(); + +#ifndef SCM_MAGIC_SNARFER +#include "mu-guile.x" +#endif /*SCM_MAGIC_SNARFER*/ + + return NULL; +} diff --git a/guile/mu-guile.hh b/guile/mu-guile.hh new file mode 100644 index 0000000..4954542 --- /dev/null +++ b/guile/mu-guile.hh @@ -0,0 +1,85 @@ +/* +** Copyright (C) 2011-2020 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_GUILE_H__ +#define __MU_GUILE_H__ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wredundant-decls" +#include +#pragma GCC diagnostic pop + +/** + * get the singleton Store instance + */ +Mu::Store& mu_guile_store(); + +/** + * whether mu-guile is initialized + * + * @return TRUE if MuGuile is Initialized, FALSE otherwise + */ +gboolean mu_guile_initialized(); + +/** + * raise a guile error (based on a GError) + * + * @param func_name function name + * @param err the error + * + * @return SCM_UNSPECIFIED + */ +SCM mu_guile_g_error(const char* func_name, GError* err); + +/** + * raise a guile error + * + * @param func_name function + * @param status err code + * @param fmt format string for error msg + * @param args params for format string + * + * @return SCM_UNSPECIFIED + */ +SCM mu_guile_error(const char* func_name, int status, const char* fmt, SCM args); + +/** + * convert a string into an SCM -- . It assumes str is in UTF8 encoding, and + * replace characters with '?' if needed. + * + * @param str a string + * + * @return a guile string or #f for empty + */ +SCM mu_guile_scm_from_string(const std::string& str); + +/** + * Initialize this mu guile module. + * + * @param data + * + * @return + */ +extern "C" { +void* mu_guile_init(void* data); +} +#endif /*__MU_GUILE_H__*/ diff --git a/guile/mu-guile.texi b/guile/mu-guile.texi new file mode 100644 index 0000000..9eae2fe --- /dev/null +++ b/guile/mu-guile.texi @@ -0,0 +1,995 @@ +\input texinfo.tex @c -*-texinfo-*- +@c %**start of header +@setfilename mu-guile.info +@settitle mu-guile user manual + +@c Use proper quote and backtick for code sections in PDF output +@c Cf. Texinfo manual 14.2 +@set txicodequoteundirected +@set txicodequotebacktick + +@documentencoding UTF-8 +@c %**end of header + +@include version.texi + +@copying +Copyright @copyright{} 2012-@value{UPDATED-YEAR} Dirk-Jan C. Binnema + +@quotation +Permission is granted to copy, distribute and/or modify this document +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A +copy of the license is included in the section entitled ``GNU Free +Documentation License.'' +@end quotation +@end copying + +@titlepage +@title @t{mu-guile} - extending @t{mu} with Guile Scheme +@subtitle version @value{VERSION} +@author Dirk-Jan C. Binnema + +@c The following two commands start the copyright page. +@page +@vskip 0pt plus 1filll +@insertcopying +@end titlepage + +@dircategory The Algorithmic Language Scheme +@direntry +* Mu-guile: (mu-guile). Guile-bindings for the mu e-mail indexer/searcher +@end direntry + +@contents + +@ifnottex +@node Top +@top mu-guile manual +@end ifnottex + +@iftex +@node Welcome to mu-guile +@unnumbered Welcome to mu-guile +@end iftex + +Welcome to @t{mu-guile}! + +@t{mu} is a program for indexing and searching your e-mails. It can search +your messages in many different ways, but sometimes that may not be +enough. If you have very specific queries, or want do generate some +statistics, you need some more power. + +@t{mu-guile} is made for those cases. @t{mu-guile} exposes the internals of +@t{mu} and its database to the @t{guile} programming language. Guile is the +@emph{GNU Ubiquitous Intelligent Language for Extensions} - a version of the +@emph{Scheme} programming language and the official GNU extension language. + +Guile/Scheme is a member of the @emph{Lisp} family of programming languages -- +like emacs-lisp, @emph{Racket}, Common Lisp. If you're not familiar with +Scheme, @t{mu-guile} is an excellent opportunity to learn a bit about! + +Trust me, it's not very hard -- and it's @emph{fun}! + +@menu +* Getting started:: +* Initializing mu-guile:: +* Messages:: +* Contacts:: +* Attachments and other parts:: +* Statistics:: +* Plotting data:: +* Writing scripts:: + +Appendices + +* Recipes:: Snippets do specific things +* GNU Free Documentation License:: The license of this manual. +@end menu + +@node Getting started +@chapter Getting started + +@menu +* Installation:: +* Making sure it works:: +@end menu + +This chapter walks you through the installation and the basic setup. + +@node Installation +@section Installation + +@t{mu-guile} is part of @t{mu} - by installing the latter, the former is +necessarily installed as well. At the time of writing, there are no +distribution-provided packaged versions of @t{mu-guile}; so for now, you need +to follow the steps below. + +@subsection Guile 2.x + +@t{mu-guile} is built automatically when @t{mu} is built, if you have +@t{guile} version 2 or higher. (@t{mu} checks for this during +@t{configure}). Thus, the first step is to ensure you have @t{guile} +installed. + +On Debian/Ubuntu you can install @t{guile} 2.x using the @t{guile-2.0-dev} +package (and its dependencies): +@example +$ sudo apt-get install guile-2.0-dev +@end example + +At the time of writing, there are no official packages for +Fedora@footnote{@url{https://bugzilla.redhat.com/show_bug.cgi?id=678238}}. If +you are using Fedora or any other system that does not have packages, you need +to compile @t{guile} from +source@footnote{@url{http://www.gnu.org/software/guile/manual/html_node/Obtaining-and-Installing-Guile.html#Obtaining-and-Installing-Guile}}. + +@subsection gnuplot + +For creating graphs with @t{mu-guile}, you need the @t{gnuplot} program -- +most likely, there is a package available for your system; for example: + +@example +$ sudo apt-get install gnuplot +@end example + +and in Fedora: + +@example +$ sudo yum install gnuplot +@end example + +@subsection mu + +Assuming @t{guile} 2.x is installed correctly, @t{mu} finds it during its +@t{configure}-stage, and creates @t{mu-guile}. Building @t{mu} follows the +normal steps -- please see the @t{mu} documentation for the details. + +The output of @t{./configure} should end with a little text describing the +detected versions of various libraries @t{mu} depends on. In particular, it +should mention the @t{guile} version, e.g. + +@example +Guile version : 2.0.3.82-a2c66 +@end example + +If you don't see any line referring to @t{guile}, please install it, and run +@t{configure} again. After a successful @t{./configure}, we can make and +install the package: + +@example +$ make && sudo make install +@end example + +@subsection mu-guile + +After this, @t{mu} and @t{mu-guile} are installed -- usually somewhere under +@t{/usr/local}.You may need to update @t{guile}'s @code{%load-path} to find it +there. You can check the current @code{%load-path} with the following: + +@example +guile -c '(display %load-path)(newline)' +@end example + +If necessary, you can add the @t{%load-path} by adding to your +@file{~/.guile}: + +@lisp +(set! %load-path (cons "/usr/local/share/guile/site/2.0" %load-path)) +@end lisp + +Or, alternatively, you can set @t{GUILE_LOAD_PATH}: +@example +export GUILE_LOAD_PATH=/usr/local/share/guile/site/2.0 +@end example + +In both cases the directory should be the directory that contains the +installed @t{mu.scm}; if you installed @t{mu} under a different prefix, you +must change the @code{%load-path} accordingly. After this, you should be ready +to go! + +Furthermore, you need to ensure that @t{guile} can find the mu-guile +library; for this we can use @code{LTDL_LIBRARY_PATH}, e.g. +@example +export LTDL_LIBRARY_PATH=/usr/local/lib +@end example + +@node Making sure it works +@section Making sure it works + +Assuming @t{mu-guile} has been installed correctly (@ref{Installation}), and +also assuming that you have already indexed your e-mail messages (if +necessary, see the @t{mu-index} man-page), we are ready to start @t{mu-guile}; +a session may look something like this: + +@cartouche +@verbatim +GNU Guile 2.0.5.123-4bd53 +Copyright (C) 1995-2012 Free Software Foundation, Inc. + +Guile comes with ABSOLUTELY NO WARRANTY; for details type `,show w'. +This program is free software, and you are welcome to redistribute it +under certain conditions; type `,show c' for details. + +Enter `,help' for help. +scheme@(guile-user)> +@end verbatim +@end cartouche + +@noindent +Now, copy-paste the following after the prompt: + +@cartouche +@lisp +(use-modules (mu)) +(mu:initialize) +(for-each + (lambda(msg) + (format #t "Subject: ~a\n" (mu:subject msg))) + (mu:message-list "hello")) +@end lisp +@end cartouche + +@noindent +After pressing @key{Enter}, you should get a list of all subjects of messages +that match @t{hello}: + +@verbatim +... +Subject: RE: The Bird Serpent War Cataclysm +Subject: Hello! +Subject: Re: post-run tomorrow +Subject: When all is lost +... +@end verbatim + +@noindent +If all this works, congratulations! @t{mu-guile} is installed now, ready to +serve your every searching need! + +@node Initializing mu-guile +@chapter Initializing mu-guile + +We now have installed @t{mu-guile}, and in @ref{Making sure it works} +confirmed that things work by trying some simple script. In this and the +following chapters, we take a closer look at programming with @t{mu-guile}. + +It is possible to write separate programs with @t{mu-guile}, but for now we'll +do things @emph{interactively}, that is, from the Guile-prompt +(``@abbr{REPL}''). + +As we have seen, we start our @t{mu-guile} session by starting @t{guile}: + +@verbatim +$ guile +@end verbatim + +@cartouche +@verbatim +GNU Guile 2.0.5.123-4bd53 +Copyright (C) 1995-2012 Free Software Foundation, Inc. + +Guile comes with ABSOLUTELY NO WARRANTY; for details type `,show w'. +This program is free software, and you are welcome to redistribute it +under certain conditions; type `,show c' for details. + +Enter `,help' for help. +scheme@(guile-user)> +@end verbatim +@end cartouche + +The first thing we need to do is loading the modules. All the basics are in +the @t{(mu)} module, with some statistical extras in @t{(mu stats)}, and some +graph plotting functionality in @t{(mu plot)}@footnote{@code{(mu plot)} +requires the @t{gnuplot} program}. Let's load all of them: +@verbatim +scheme@(guile-user)> (use-modules (mu) (mu stats) (mu plot)) +@end verbatim + +The first time you do this, @t{guile} will probably respond by showing some +messages about compiling the modules, and then return to you with another +prompt. Before we can do anything with @t{mu guile}, we need to initialize the +system: + +@verbatim +scheme@(guile-user)> (mu:initialize) +@end verbatim + +This opens the database for reading, using the default location of +@file{~/.cache/mu}@footnote{If you keep your @t{mu} database in a non-standard +place, use @code{(mu:initialize "/path/to/my/mu/")}} + +Now, @t{mu-guile} is ready to go. In the next chapter, we go through the +modules and show what you can do with them. + +@node Messages +@chapter Messages + +In this chapter, we discuss searching messages and doing things with them. + +@menu +* Finding messages:: query for messages in the database +* Message methods:: what methods are available for messages? +* Example - the longest subject:: find the messages with the longest subject +@end menu + +@node Finding messages +@section Finding messages +Now we are ready to retrieve some messages from the system. There are two main +procedures to do this: + +@itemize +@item @code{(mu:message-list [])} +@item @code{(mu:for-each-message [])} +@end itemize + +@noindent +The first procedure, @code{mu:message-list} returns a list of all messages +matching @t{}; if you leave @t{} out, it +returns @emph{all} messages. For example, to get all messages with @t{coffee} +in the subject line: + +@verbatim +scheme@(guile-user)> (mu:message-list "subject:coffee") +$1 = (#< 9040640> #< 9040630> + #< 9040570>) +@end verbatim + +@noindent +Apparently, we have three messages matching @t{subject:coffee}, so we get a +list of three @code{} objects. Let's just use the +@code{mu:subject} procedure ('method') provided by @code{} objects +to retrieve the subject-field (more about methods in the next section). + +For your convenience, @t{guile} has saved the result of our last query in a +variable called @t{$1}, so to get the subject of the first message in the +list, we can do: + +@verbatim +scheme@(guile-user)> (mu:subject (car $1)) +$2 = "Re: best coffee ever!" +@end verbatim + +@noindent +The second procedure we mentioned, @code{mu:for-each-message}, executes some +procedure for each message matched by the search expression (or @emph{all} +messages if the search expression is omitted): + +@verbatim +scheme@(guile-user)> (mu:for-each-message + (lambda(msg) + (display (mu:subject msg)) + (newline)) + "subject:coffee") +Re: best coffee ever! +best coffee ever! +Coffee beans +scheme@(guile-user)> +@end verbatim + +@noindent +Using @code{mu:message-list} and/or +@code{mu:for-each-message}@footnote{Implementation node: +@code{mu:message-list} is implemented in terms of @code{mu:for-each-message}, +not the other way around. Due to the way @t{mu} works, +@code{mu:for-each-message} is rather more efficient than a combination of +@code{for-each} and @code{mu:message-list}} and a couple of @t{} +methods, together with what Guile/Scheme provides, should allow for many +interesting programs. + +@node Message methods +@section Message methods + +Now that we've seen how to retrieve lists of message objects +(@code{}), let's see what we can do with such an object. + +@code{} defines the following methods that all take a single +@code{} object as a parameter. We won't go into the exact meanings +for all of these procedures here - for the details about various flags / +properties, please refer to the @t{mu-find} man-page. + +@itemize +@item @code{(mu:bcc msg)}: the @t{Bcc} field of the message, or @t{#f} if there is none +@item @code{(mu:body-html msg)}: : the html body of the message, or @t{#f} if there is none +@item @code{(mu:body-txt msg)}: the plain-text body of the message, or @t{#f} if there is none +@item @code{(mu:cc msg)}: the @t{Bcc} field of the message, or @t{#f} if there is none +@item @code{(mu:date msg)}: the @t{Date} field of the message, or 0 if there is none +@item @code{(mu:flags msg)}: list of message-flags for this message +@item @code{(mu:from msg)}: the @t{From} field of the message, or @t{#f} if there is none +@item @code{(mu:maildir msg)}: the maildir this message lives in, or @t{#f} if there is none +@item @code{(mu:message-id msg)}: the @t{Message-Id} field of the message, or @t{#f} if there is none +@item @code{(mu:path msg)}: the file system path for this message +@item @code{(mu:priority msg)}: the priority of this message (either @t{mu:prio:low}, @t{mu:prio:normal} or @t{mu:prio:high} +@item @code{(mu:references msg)}: the list of messages (message-ids) this message +refers to in(mu: the @t{References:} header +@item @code{(mu:size msg)}: size of the message in bytes +@item @code{(mu:subject msg)}: the @t{Subject} field of the message, or @t{#f} if there is none. +@item @code{(mu:tags msg)}: list of tags for this message +@item @code{(mu:timestamp msg)}: the timestamp (mtime) of the message file, or +#f if there is none. +message file +@item @code{(mu:to msg)}: the sender of the message, or @t{#f} if there is none +@end itemize + +With these methods, we can query messages for their properties; for example: + +@verbatim +scheme@(guile-user)> (define msg (car (mu:message-list "snow"))) +scheme@(guile-user)> (mu:subject msg) +$1 = "Re: Running in the snow is beautiful" +scheme@(guile-user)> (mu:flags msg) +$2 = (mu:flag:replied mu:flag:seen) +scheme@(guile-user)> (strftime "%F" (localtime (mu:date msg))) +$3 = "2011-01-15" +@end verbatim + +There are a couple more methods: +@itemize +@item @code{(mu:header msg "")} returns an arbitrary message +header (or @t{#f} if not found) -- e.g. @code{(header msg "User-Agent")} +@item If you include the @t{mu contact} module, the @code{(mu:contacts +msg [contact-type])} method (to get a list of contacts) is +added. @xref{Contacts}. +@item If you include the @t{mu part} module, the @code{((mu:parts msg)} and +@code{(mu:attachments msg)} methods are added. @xref{Attachments and other parts}. +@end itemize + +@node Example - the longest subject +@section Example - the longest subject + +Now, let's write a little example -- let's find out what is the @emph{longest +subject} of any e-mail messages we received in the year 2011. You can try +this if you put the following in a separate file, make it executable, and run +it like any program. + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu)) +(use-modules (srfi srfi-1)) + +(mu:initialize) + +;; note: (subject msg) => #f if there is no subject +(define list-of-subjects + (map (lambda (msg) + (or (mu:subject msg) "")) (mu:message-list "date:2011..2011"))) +;; see the mu-find manpage for the date syntax + +(define longest-subject + (fold (lambda (subj1 subj2) + (if (> (string-length subj1) (string-length subj2)) + subj1 subj2)) + "" list-of-subjects)) + +(format #t "Longest subject: ~s\n" longest-subject) +@end lisp + +There are many other ways to solve the same problem, for example by using an +iterative approach with @code{mu:for-each-message}, but it should show how one +can easily write little programs to answer specific questions about your +e-mail corpus. + +@node Contacts +@chapter Contacts + +We can retrieve the sender and recipients of an e-mail message using methods +like @code{mu:from}, @code{mu:to} etc.; @xref{Message methods}. These +procedures return the list of recipients as a single string; however, often it +is more useful to deal with recipients as separate objects. + +@menu +* Contact procedures and objects:: +* All contacts:: +* Utility procedures:: +* Example - mutt export:: +@end menu + + +@node Contact procedures and objects +@section Contact procedures and objects + +Message objects (@pxref{Messages}) have a method @t{mu:contacts}: + + @code{(mu:contacts [])} + +The @t{} is a symbol, one of @code{mu:to}, @code{mu:from}, +@code{mu:cc} or @code{mu:bcc}. This will then get the contact objects for the +contacts of the corresponding type. If you leave out the contact-type (or +specify @t{#t} for it, you will get a list of @emph{all} contact objects for +the message. + +A contact object (@code{}) has two methods: +@itemize +@item @code{mu:name} returns the name of the contact, or #f if there is none +@item @code{mu:email} returns the e-mail address of the contact, or #f if there is none +@end itemize + +Let's get a list of all names and e-mail addresses in the 'To:' field, of +messages matching 'book': + +@lisp +(use-modules (mu)) +(mu:initialize) +(mu:for-each-message + (lambda (msg) + (for-each + (lambda (contact) + (format #t "~a => ~a\n" + (or (mu:email contact) "") (or (mu:name contact) "no-name"))) + (mu:contacts msg mu:contact:to))) + "book") +@end lisp + +This shows what the methods do, but for many uses, it would be more useful to +have each of the contacts only show up @emph{once} - for that, please refer to +@xref{All contacts}. + +@node All contacts +@section All contacts + +Sometimes you may want to inspect @emph{all} the different contacts in the +@t{mu} database. This is useful, for instance, when exporting contacts to some +external format that can then be important in an e-mail program. + +To enable this, there is the procedure @code{mu:for-each-contact}, defined as + + @code{(mu:for-each-contact procedure [search-expression])}. + +This will aggregate the unique contacts from @emph{all} messages matching +@t{} (when it is left empty, it will match all messages in +the database), and execute @t{procedure} for each of them. + +The @t{procedure} receives an object of the type @t{}, +which is a @emph{subclass} of the @t{} class discussed in +@xref{Contact procedures and objects}. @t{} objects +expose the following additional methods: + +@itemize +@item @code{(mu:frequency )}: returns the @emph{number of times} this contact occurred in +one of the address fields +@item @code{(mu:last-seen )}: returns the @emph{most recent time} the contact was +seen in one of the address fields, as a @t{time_t} value +@end itemize + +The method assumes an e-mail address is unique for a certain contact; if a +certain e-mail address occurs with different names, it uses the most recent +non-empty name. + +@node Utility procedures +@section Utility procedures + +To make dealing with contacts even easier, there are a number of utility +procedures that can save you a bit of typing. + +For converting contacts to some textual form, there is @code{(mu:contact->string + format)}, which takes a contact and returns a text string with +the given format. Currently supported formats are @t{"org-contact}, @t{"mutt-alias"}, +@t{"mutt-ab"}, @t{"wanderlust"} and @t{"plain"}. + + +@node Example - mutt export +@section Example - mutt export + +Let's see how we could export the addresses in the @t{mu} database to the +addressbook format of the venerable +@t{mutt}@footnote{@url{http://www.mutt.org/}} e-mail client. + +The addressbook format that @t{mutt} uses is a sequence of lines that look +something like: +@verbatim +alias [] "<" ">" +@end verbatim + +@t{mu guile} provides the procedure @code{(mu:contact->string +format)} that we can use to do the conversion. + +We may want to focus on people with whom we have frequent correspondence; so +we may want to limit ourselves to people we have seen at least 10 times in the +last year. + +It is a bit hard to @emph{guess} the nick name for e-mail contacts, but +@code{mu:contact->string} tries something based on the name. You can always +adjust them later by hand, obviously. + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu)) +(mu:initialize) + +;; Get a list of contacts that were seen at least 20 times since 2010 +(define (selected-contacts) + (let ((addrs '()) + (start (car (mktime (car (strptime "%F" "2010-01-01"))))) + (minfreq 20)) + (mu:for-each-contact + (lambda (contact) + (if (and (mu:email contact) + (>= (mu:frequency contact) minfreq) + (>= (mu:last-seen contact) start)) + (set! addrs (cons contact addrs))))) + addrs)) + +(for-each + (lambda (contact) + (format #t "~a\n" (mu:contact->string contact "mutt-alias"))) + (selected-contacts)) +@end lisp + +This simple program could be improved in many ways; this is left as an +exercise to the reader. + +@node Attachments and other parts +@chapter Attachments and other parts + +To deal with @emph{attachments}, or, more in general @emph{MIME-parts}, there +is the @t{mu part} module. + +@menu +* Parts and their methods:: +* Attachment example:: +@end menu + +@node Parts and their methods +@section Parts and their methods +The module defines the @code{} class, and adds two methods to +@code{} objects: +@itemize +@item @code{(mu:parts msg)} - returns a list @code{} objects, one for +each MIME-parts in the message. +@item @code{(mu:attachments msg)} - like @code{parts}, but only list those MIME-parts +that look like proper attachments. +@end itemize + +A @code{} object exposes a few methods to get information about the +part: +@itemize +@item @code{(mu:name )} - returns the file name of the mime-part, or @code{#f} if +there is none. +@item @code{(mu:mime-type )} - returns the mime-type of the mime-part, or @code{#f} +if there is none. +@item @code{(mu:size )} - returns the size in bytes of the mime-part +@end itemize + +@c Then, we may want to save the part to a file; this can be done using either: +@c @itemize +@c @item @code{(mu:save part )} - save a part to a temporary file, return the file +@c name@footnote{the temporary filename is a predictable procedure of (user-id, +@c msg-path, part-index)} +@c @item @code{(mu:save-as )} - save part to file at path +@c @end itemize + +@node Attachment example +@section Attachment example + +Let's look at some small example. Let's get a list of the biggest attachments +in messages about Luxemburg: + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu)) +(mu:initialize) + +(define (all-attachments expr) + "Return a list of (name . size) for all attachments in messages +matching EXPR." + (let ((pairs '())) + (mu:for-each-message + (lambda (msg) + (for-each + (lambda (att) ;; add (filename . size) to the list + (set! pairs (cons (cons (mu:name att) (or (mu:size att) 0)) pairs))) + (mu:attachments msg))) + expr) + pairs)) + +(for-each + (lambda (att) + (format #t "~a: ~,1fKb\n" + (car att) (exact->inexact (/ (cdr att) 1024)))) + (sort (all-attachments "Luxemburg") + (lambda (att1 att2) + (< (cdr att1) (cdr att2))))) +@end lisp + +As an exercise for the reader, you might want to re-rewrite the +@code{all-attachments} in terms of @code{mu:message-list}, which would +probably be a bit more elegant. + + +@node Statistics +@chapter Statistics + +@t{mu-guile} offers some convenience procedures to determine various statistics +about the messages in the database. + +@menu +* Basics:: @code{mu:count}, @code{mu:average}, ... +* Tabulating values:: @code{mu:tabulate} +* Most frequent values:: @code{mu:top-n-most-frequent} +@end menu + +@node Basics +@section Basics + +Let's look at some of the basic statistical operations available, in an +interactive session: +@example +GNU Guile 2.0.5.123-4bd53 +Copyright (C) 1995-2012 Free Software Foundation, Inc. + +Guile comes with ABSOLUTELY NO WARRANTY; for details type `,show w'. +This program is free software, and you are welcome to redistribute it +under certain conditions; type `,show c' for details. + +Enter `,help' for help. +scheme@@(guile-user)> ;; load modules, initialize mu +scheme@@(guile-user)> (use-modules (mu) (mu stats)) +scheme@@(guile-user)> (mu:initialize) +scheme@@(guile-user)> +scheme@@(guile-user)> ;; count the number of messages with 'hello' in their subject +scheme@@(guile-user)> (mu:count "subject:hello") +$1 = 162 +scheme@@(guile-user)> ;; average the size of messages with hello in their subject +scheme@@(guile-user)> (mu:average mu:size "subject:hello") +$2 = 34597733/81 +scheme@@(guile-user)> (exact->inexact $2) +$3 = 427132.506172839 +scheme@@(guile-user)> ;; calculate the correlation between message size and +scheme@@(guile-user)> ;; subject length +scheme@@(guile-user)> (mu:correl mu:size (lambda (msg) + (string-length (mu:subject msg))) "subject:hello") +$5 = -0.10804368622292 +scheme@@(guile-user)> +@end example + +@node Tabulating values +@section Tabulating values + +@code{(mu:tabulate [])} applies @t{} to each +message matching @t{} (leave empty to match @emph{all} messages), +and returns a associative list (a list of pairs) with each of the different +results of @t{} and their frequencies. For fields that contain lists +of values (such as address-fields), each of the values in the list is added +separately. + +@subsection Example: messages per weekday + +We demonstrate @code{mu:tabulate} with an example. Suppose we want to know how +many messages we receive per weekday: + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu) (mu stats) (mu plot)) +(mu:initialize) + +;; create a list like (("Sun" . 13) ("Mon" . 23) ...) +(define weekday-table + (mu:weekday-numbers->names + (sort + (mu:tabulate + (lambda (msg) + (tm:wday (localtime (mu:date msg))))) + (lambda (a b) (< (car a) (car b)))))) + +(for-each + (lambda (elm) + (format #t "~a: ~a\n" (car elm) (cdr elm))) + weekday-table) +@end lisp + + +The procedure @code{weekday-table} uses @code{mu:tabulate-message} to get the +frequencies per hour -- this returns a list of pairs: +@verbatim +((5 . 2339) (0 . 2278) (4 . 2800) (2 . 3184) (6 . 1856) (3 . 2833) (1 . 2993)) +@end verbatim + +We sort these pairs by the day number, and then apply +@code{mu:weekday-numbers->names}, which takes the list, and returns a list +where the day numbers are replace by there abbreviated name (in the current +locale). Note, there is also @code{mu:month-numbers->names}. + +The script then outputs these numbers in the following form: + +@verbatim +Sun: 2278 +Mon: 2993 +Tue: 3184 +Wed: 2833 +Thu: 2800 +Fri: 2339 +Sat: 1856 +@end verbatim + +Clearly, Saturday is a slow day for e-mail... + +@node Most frequent values +@section Most frequent values + +In the above example, the number of values is small (the seven weekdays); +however, in many cases there can be many different values (for example, all +different message subjects), many of which may not be very interesting -- all +we need to know is the top-10 of most frequently seen values. + +This is fairly easy to achieve using @code{mu:tabulate} -- to get the top-10 +subjects@footnote{this requires the @code{(srfi srfi-1)}-module}, we can use +something like this: +@lisp +(take + (sort + (mu:tabulate mu:subject) + (lambda (a b) (> (cdr a) (cdr b)))) + 10) +@end lisp + +If this is not short enough, @t{mu-guile} offers a convenience procedure to do +this: @code{mu:top-n-most-frequent}. For example, to get the top-10 people we +sent mail to most often: + +@lisp +(mu:top-n-most-frequent mu:to 10 "maildir:/sent") +@end lisp + +Can't make it much easier than that! + + +@node Plotting data +@chapter Plotting data + +You can plot the results in the format produced by @code{mu:tabulate} with the +@t{(mu plot)} module, an experimental module that requires the +@t{gnuplot}@footnote{@url{http://www.gnuplot.info/}} program to be installed +on your system. + +The @code{mu:plot-histogram} procedure takes the following arguments: + +@code{(mu:plot-histogram <x-label> <y-label> [<want-ascii>])} + +Here, @code{<data>} is a table of data in the format that @code{mu:tabulate} +produces. @code{<title>}, @code{<x-label>} and @code{<y-lablel>} are, +respectively, the title of the graph, and the labels for X- and +Y-axis. Finally, if you pass @t{#t} for the final @code{<want-ascii>} +parameter, a plain-text rendering of the graph will be produced; otherwise, a +graphical window will be shown. + +An example should clarify how this works in practice; let's plot the number of +message per hour: + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu) (mu stats) (mu plot)) +(mu:initialize) + +(define (mail-per-hour-table) + (sort + (mu:tabulate + (lambda (msg) + (tm:hour (localtime (mu:date msg))))) + (lambda (x y) (< (car x) (car y))))) + +(mu:plot-histogram (mail-per-hour-table) "Mail per hour" "Hour" "Frequency") +@end lisp + +@cartouche +@verbatim + Mail per hour +Frequency + 1200 ++--+--+--+--+-+--+--+--+--+-+--+--+--+-+--+--+--+--+-+--+--+--+--++ + |+ + + + + + + "/tmp/fileHz7D2u" using 2:xticlabels(1) ******** + 1100 ++ *** +* + **** * * * + 1000 *+ * **** * +* + * * ****** **** * ** * * + 900 *+ * * ** **** * **** ** * +* + * * * ** * * ********* * ** ** * * + 800 *+ * **** ** * * * * ** * * ** ** * +* + 700 *+ *** **** * ** * * * * ** **** * ** ** * +* + * * * **** * * ** * * * * ** * **** ** ** * * + 600 *+ * **** * * * * ** * * * * ** * * * ** ** * +* + * * ** * * * * * ** * * * * ** * * * ** ** * * + 500 *+ * ** * * * * * ** * * * * ** * * * ** ** * +* + * * ** **** *** * * * ** * * * * ** * * * ** ** * * + 400 *+ * ** ** **** * * * * * ** * * * * ** * * * ** ** * +* + *+ *+**+**+* +*******+* +* +*+ *+**+* +*+ *+ *+**+* +*+ *+**+**+* +* + 300 ******************************************************************** + 0 1 2 3 4 5 6 7 8 910 11 12 1314 15 16 17 1819 20 21 22 23 + Hour +@end verbatim +@end cartouche + +@node Writing scripts +@chapter Writing scripts + +The @t{mu} program has built-in support for running guile-scripts, and comes +with a number of examples. + +You can get a list of all scripts with the @t{mu script} command: +@verbatim +$ mu script +Available scripts (use --verbose for details): + * find-dups: find duplicate messages + * msgs-count: count the number of messages matching some query + * msgs-per-day: graph the number of messages per day + * msgs-per-hour: graph the number of messages per hour + * msgs-per-month: graph the number of messages per month + * msgs-per-year: graph the number of messages per year + * msgs-per-year-month: graph the number of messages per year-month +@end verbatim + +You can then execute such a script by its name: +@verbatim +$ mu msgs-per-month --textonly --query=hello + + + Messages per month matching hello + + 240 ++-+-----+----+-----+-----+-----+----+-----+-----+-----+----+-----+-++ + | + + + + "/tmp/filewi9H0N" using 2:xticlabels(1) ****** | + 220 ++ * * ****** + | * * * * + 200 ++ * * * +* + | * * * * + 180 ++ ****** * * * +* + | * * * * * * + 160 ****** * * * * * +* + * * * * * * * * + * ******* * * * * ****** * * + 140 *+ ** * * * * * * ******** +* + * ** ******* * * * * * ** ** * + 120 *+ ** ** ******* * * * * ** ** +* + * ** ** ** * * * ******* ** ** * + 100 *+ ** ** ** * * * * ** ** ** +* + * + ** + ** + ** + * + * + + * + * + ** + ** + ** + * + 80 ********************************************************************** + Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec + Month +@end verbatim + +Please refer to the @t{mu-script} man-page for some details on writing your +own scripts. + + +@node Recipes +@appendix Recipes + +@itemize +@item Calculating the average length of subject-lines +@lisp +;; the average length of all our +(let ((len 0) (n 0)) + (mu:for-each-message + (lambda (msg) + (set! len (+ len (string-length (or (mu:subject msg) "")))) + (set! n (+ n 1)))) + (if (= n 0) 0 (/ len n))) + ;; this gives a rational, exact result; + ;; use exact->inexact to get decimals + +;; we we can make this short with the mu:average (with (mu stats)) +(mu:average (lambda (msg) (string-length (or (mu:subject msg) "")))) + + +@end lisp +@end itemize + +@node GNU Free Documentation License +@appendix GNU Free Documentation License + +@include fdl.texi +@bye diff --git a/guile/mu-guile.x b/guile/mu-guile.x new file mode 100644 index 0000000..8aa8020 --- /dev/null +++ b/guile/mu-guile.x @@ -0,0 +1,4 @@ +/* cpp arguments: mu-guile.cc -DHAVE_CONFIG_H -I. -I.. -I../lib -I/usr/local/include/guile/3.0 -pthread -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -I/usr/include/sysprof-4 -I/usr/include/libmount -I/usr/include/blkid -pthread -fno-strict-aliasing -Wall -Wextra -Wundef -Wwrite-strings -Wpointer-arith -Wmissing-declarations -Wredundant-decls -Wno-unused-parameter -Wno-missing-field-initializers -Wformat=2 -Wcast-align -Wformat-nonliteral -Wformat-security -Wsign-compare -Wstrict-aliasing -Wshadow -Winline -Wpacked -Wmissing-format-attribute -Wmissing-noreturn -Winit-self -Wmissing-include-dirs -Wunused-but-set-variable -Warray-bounds -Wreturn-type -Wno-overloaded-virtual -Wswitch-enum -Wswitch-default -Wno-error=unused-parameter -Wno-error=missing-field-initializers -Wno-error=overloaded-virtual -Wno-redundant-decls -Wno-missing-declarations -Wno-suggest-attribute=noreturn -O2 -Wno-inline */ +scm_c_define_gsubr (s_mu_initialize, 0, 1, 0, (scm_t_subr) mu_initialize); scm_c_export (s_mu_initialize, __null );; +scm_c_define_gsubr (s_mu_initialized_p, 0, 0, 0, (scm_t_subr) mu_initialized_p); scm_c_export (s_mu_initialized_p, __null );; +scm_c_define_gsubr (s_log_func, 1, 0, 1, (scm_t_subr) log_func);; diff --git a/guile/mu.scm b/guile/mu.scm new file mode 100644 index 0000000..08eae1f --- /dev/null +++ b/guile/mu.scm @@ -0,0 +1,318 @@ +;; Copyright (C) 2011-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +(define-module (mu) + :use-module (oop goops) + :use-module (ice-9 optargs) + :use-module (texinfo string-utils) + :export + ( ;; classes + <mu:message> + <mu:contact> + <mu:part> + ;; general +;; mu:initialize + ;; mu:initialized? + mu:log-warning + mu:log-message + mu:log-critical + ;; search funcs + mu:for-each-message + mu:for-each-msg + mu:message-list + ;; message funcs + mu:header + ;; message accessors + mu:field:bcc + mu:field:body-html + mu:field:body-txt + mu:field:cc + mu:field:date + mu:field:flags + mu:field:from + mu:field:maildir + mu:field:message-id + mu:field:path + mu:field:prio + mu:field:refs + mu:field:size + mu:field:subject + mu:field:tags + mu:field:timestamp + mu:field:to + ;; contact funcs + mu:name + mu:email + mu:contact->string + ;; + mu:for-each-contact + ;; + mu:contacts + ;; + ;; <mu:contact-with-stats> + mu:frequency + mu:last-seen + ;; parts + + <mu:part> + ;; message function + mu:attachments + mu:parts + ;; <mu:part> methods + mu:name + mu:mime-type + ;; size + ;; mu:save + ;; mu:save-as + )) + +;; this is needed for guile < 2.0.4 +(setlocale LC_ALL "") + +;; load the binary +(load-extension "libguile-mu" "mu_guile_init") +(load-extension "libguile-mu" "mu_guile_message_init") + +;; define some dummies so we don't get errors during byte compilation +(eval-when (compile) + (define mu:c:get-field) + (define mu:c:get-contacts) + (define mu:c:for-each-message) + (define mu:c:get-header) + (define mu:critical) + (define mu:c:log) + (define mu:message) + (define mu:c:log) + (define mu:warning) + (define mu:c:log) + (define mu:c:get-parts)) + +(define (mu:log-warning frm . args) + "Log FRM with ARGS at warning." + (mu:c:log mu:warning frm args)) + +(define (mu:log-message frm . args) + "Log FRM with ARGS at warning." + (mu:c:log mu:message frm args)) + +(define (mu:log-critical frm . args) + "Log FRM with ARGS at warning." + (mu:c:log mu:critical frm args)) + +(define-class <mu:message> () + (msg #:init-keyword #:msg)) ;; the MuMsg-smob we're wrapping + +(define-syntax define-getter + (syntax-rules () + ((define-getter method-name field) + (begin + (define-method (method-name (msg <mu:message>)) + (mu:c:get-field (slot-ref msg 'msg) field)) + (export method-name))))) + +(define-getter mu:bcc mu:field:bcc) +(define-getter mu:body-html mu:field:body-html) +(define-getter mu:body-txt mu:field:body-txt) +(define-getter mu:cc mu:field:cc) +(define-getter mu:date mu:field:date) +(define-getter mu:flags mu:field:flags) +(define-getter mu:from mu:field:from) +(define-getter mu:maildir mu:field:maildir) +(define-getter mu:message-id mu:field:message-id) +(define-getter mu:path mu:field:path) +(define-getter mu:priority mu:field:prio) +(define-getter mu:references mu:field:refs) +(define-getter mu:size mu:field:size) +(define-getter mu:subject mu:field:subject) +(define-getter mu:tags mu:field:tags) +(define-getter mu:timestamp mu:field:timestamp) +(define-getter mu:to mu:field:to) + +(define-method (mu:header (msg <mu:message>) (hdr <string>)) + "Get an arbitrary header HDR from message MSG; return #f if it does +not exist." + (mu:c:get-header (slot-ref msg 'msg) hdr)) + +(define* (mu:for-each-message func #:optional (expr #t) (maxresults -1)) + "Execute function FUNC for each message that matches mu search expression EXPR. +If EXPR is not provided, match /all/ messages in the store. MAXRESULTS +specifies the maximum of messages to return, or -1 (the default) for +no limit." + (mu:c:for-each-message + (lambda (msg) + (func (make <mu:message> #:msg msg))) + expr + maxresults)) + +;; backward-compatibility alias +(define mu:for-each-msg mu:for-each-message) + +(define* (mu:message-list #:optional (expr #t) (maxresults -1)) + "Return a list of all messages matching mu search expression +EXPR. If EXPR is not provided, return a list of /all/ messages in the +store. MAXRESULTS specifies the maximum of messages to return, or +-1 (the default) for no limit." + (let ((lst '())) + (mu:for-each-message + (lambda (m) + (set! lst (append! lst (list m)))) expr maxresults) + lst)) + +;; contacts +(define-class <mu:contact> () + (name #:init-value #f #:accessor mu:name #:init-keyword #:name) + (email #:init-value #f #:accessor mu:email #:init-keyword #:email)) + +(define-method (mu:contacts (msg <mu:message>) contact-type) + "Get all contacts for MSG of the given CONTACT-TYPE. MSG is of type <mu-message>, +while contact type is either `mu:contact:to', `mu:contact:cc', +`mu:contact:from' or `mu:contact:bcc' to get the corresponding type of +contacts, or #t to get all. + +Returns a list of <mu-contact> objects." + (map (lambda (pair) ;; a pair (na . addr) + (make <mu:contact> #:name (car pair) #:email (cdr pair))) + (mu:c:get-contacts (slot-ref msg 'msg) contact-type))) + +(define-method (mu:contacts (msg <mu:message>)) + "Get contacts of all types for message MSG as a list of <mu-contact> +objects." + (mu:contacts msg #t)) + +(define-class <mu:contact-with-stats> (<mu:contact>) + (tstamp #:init-value 0 #:accessor mu:timestamp #:init-keyword #:timestamp) + (last-seen #:init-value 0 #:accessor mu:last-seen) + (freq #:init-value 1 #:accessor mu:frequency)) + +(define* (mu:for-each-contact proc #:optional (expr #t)) + "Execute PROC for each contact. PROC receives a <mu-contact> instance +as parameter. If EXPR is specified, only consider contacts in messages +matching EXPR." + (let ((c-hash (make-hash-table 4096))) + (mu:for-each-message + (lambda (msg) + (for-each + (lambda (ct) + (let ((ct-ws (make <mu:contact-with-stats> + #:name (mu:name ct) + #:email (mu:email ct) + #:timestamp (mu:date msg)))) + (update-contacts-hash c-hash ct-ws))) + (mu:contacts msg #t))) + expr) + (hash-for-each ;; c-hash now contains a map of email->contact + (lambda (email ct-ws) (proc ct-ws)) c-hash))) + +(define-method (update-contacts-hash c-hash (nc <mu:contact-with-stats>)) + "Update the contacts hash with a new and/or existing contact." + ;; xc: existing-contact, nc: new contact + (let ((xc (hash-ref c-hash (mu:email nc)))) + (if (not xc) ;; no existing contact with this email address? + (hash-set! c-hash (mu:email nc) nc) ;; store the new contact. + ;; otherwise: + (begin + ;; 1) update the frequency for the existing contact + (set! (mu:frequency xc) (1+ (mu:frequency xc))) + ;; 2) update the name if the new one is not empty and its timestamp is newer + ;; in that case, also update the timestamp + (if (and (mu:name nc) (> (string-length (mu:name nc))) + (> (mu:timestamp nc) (mu:timestamp xc))) + (set! (mu:name xc) (mu:name nc)) + (set! (mu:timestamp xc) (mu:timestamp nc))) + ;; 3) update last-seen with timestamp, if x's timestamp is newer + (if (> (mu:timestamp nc) (mu:last-seen xc)) + (set! (mu:last-seen xc) (mu:timestamp nc))) + ;; okay --> now xc has been updated; but it back in the hash + (hash-set! c-hash (mu:email xc) xc))))) + +(define-method (mu:contact->string (contact <mu:contact>) (form <string>)) + "Convert a contact to a string in format FORM, which is a string, +either \"org-contact\", \"mutt-alias\", \"mutt-ab\", +\"wanderlust\", \"quoted\" \"plain\"." + (let* ((name (mu:name contact)) (email (mu:email contact)) + (nick ;; simplistic nick guessing... + (string-map + (lambda(kar) + (if (char-alphabetic? kar) kar #\_)) + (string-downcase (or name email))))) + (cond + ((string= form "plain") + (format #f "~a~a~a" (or name "") (if name " " "") email)) + ((string= form "org-contact") + (format #f "* ~s\n:PROPERTIES:\n:EMAIL:~a\n:NICK:~a\n:END:" + (or name email) email nick)) + ((string= form "wanderlust") + (format #f "~a ~s ~s" + nick (or name email) email)) + ((string= form "mutt-alias") + (format #f "alias ~a ~a <~a>" + nick (or name email) email)) + ((string= form "mutt-ab") + (format #f "~a\t~a\t" + email (or name ""))) + ((string= form "quoted") + (string-append + "\"" + (escape-special-chars + (string-append + (if name + (format #f "\"~a\" " name) + "") + (format #f "<~a>" email)) + "\"" #\\) + "\"")) + (else (error "Unsupported format"))))) + +;; message parts + + +(define-class <mu:part> () + (msgpath #:init-value #f #:init-keyword #:msgpath) + (index #:init-value #f #:init-keyword #:index) + (name #:init-value #f #:getter mu:name #:init-keyword #:name) + (mime-type #:init-value #f #:getter mu:mime-type #:init-keyword #:mime-type) + (size #:init-value 0 #:getter mu:size #:init-keyword #:size)) + +(define-method (get-parts (msg <mu:message>) (files-only <boolean>)) + "Get the part for MSG as a list of <mu:part> objects; if FILES-ONLY is #t, +only get the part with file names." + (map (lambda (part) + (make <mu:part> + #:msgpath (list-ref part 0) + #:index (list-ref part 1) + #:name (list-ref part 2) + #:mime-type (list-ref part 3) + #:size (list-ref part 4))) + (mu:c:get-parts (slot-ref msg 'msg) files-only))) + +(define-method (mu:attachments (msg <mu:message>)) + "Get the attachments for MSG as a list of <mu:part> objects." + (get-parts msg #t)) + +(define-method (mu:parts (msg <mu:message>)) + "Get the MIME-parts for MSG as a list of <mu-part> objects." + (get-parts msg #f)) + +;; (define-method (mu:save (part <mu:part>)) +;; "Save PART to a temporary file, and return the file name. If the +;; part had a filename, the temporary file's file name will be just that; +;; otherwise a name is made up." +;; (mu:save-part (slot-ref part 'msgpath) (slot-ref part 'index))) + +;; (define-method (mu:save-as (part <mu:part>) (filepath <string>)) +;; "Save message-part PART to file system path PATH." +;; (copy-file (save part) filepath)) diff --git a/guile/mu/README b/guile/mu/README new file mode 100644 index 0000000..634ad8b --- /dev/null +++ b/guile/mu/README @@ -0,0 +1,207 @@ +* OUTDATED * + +* README + +** What is muile? + + `muile' is a little experiment/toy using the equally experimental mu guile + bindings, to be found in libmuguile/ in the top-level source directory. + + `guile'[1] is an interpreter/library for the Scheme programming language[2], + specifically meant for extending other programs. It is, in fact, the + official GNU language for doing so. 'muile' requires guile 2.x to get the full + support. + + Older versions may not support e.g. the 'mu-stats.scm' things discussed below. + + The combination of mu + guile is called `muile', and allows you to write + little Scheme-programs to query the mu-database, or inspect individual + messages. It is still in an experimental stage, but useful already. + +** How do I get it? + + The git-version and the future 0.9.7 version of mu will automatically build + muile if you have guile. I've been using guile 2.x from git, but installing + the 'guile-1.8-dev' package (Ubuntu/Debian) should do the trick. (I only did + very minimal testing with guile 1.8 though). + + Then, configure mu. The configure output should tell you about whether guile + was found (and where). If it's found, build mu, and toys/muile should be + created, as well. + +** What can I do with it? + + Go to toys/muile and start muile. You'll end up with a guile-shell where you + can type scheme [1], it looks something like this (for guile 2.x): + + ,---- + | scheme@(guile-user)> + `---- + + Now, let's load a message (of course, replace with a message on your system): + + ,---- + | scheme@(guile-user)> (define msg (mu:msg:make-from-file "/home/djcb/Maildir/cur/12131e7b20a2:2,S")) + `---- + + This defines a variable 'msg', which holds some message on your file + system. It's now easy to inspect this message: + + ,---- + | scheme@(guile-user)> (define msg (mu:msg:make-from-file "/home/djcb/Maildir/cur/12131e7b20a2:2,S")) + `---- + + Now, we can inspect this message a bit: + ,---- + | scheme@(guile-user)> (mu:msg:subject msg) + | $1 = "See me in bikini :-)" + | scheme@(guile-user)> (mu:msg:flags msg) + | $2 = (mu:attach mu:unread) + `---- + + and so on. Note, it's probably easiest to explore the various mu: methods + using autocompletion; to enable that make sure you have + + + ,---- + | (use-modules (ice-9 readline)) + | (activate-readline) + `---- + + in your ~/.guile configuration. + +** does this tool have some parameters? + + Yes, there is --muhome to set a non-default place for the message database + (see the documentation on --muhome in the mu-find manpage). + + And there is --msg=<path> where you specify some particular message file; + it will be available as 'mu:current-msg' in the guile (muile) environment. For + example: + + ,---- + | ./muile --msg=~/Maildir/inbox/cur/1311310172_1234:2,S + | [...] + | scheme@(guile-user)> mu:current-msg + | $1 = #<msg /home/djcb/Maildir/inbox/cur/1311310172_1234:2,S> + | scheme@(guile-user)> (mu:msg:size mu:current-msg) + | $2 = 7206 + `---- + +** What about searching messages in the database? + + That's easy, too - it does require a little more scheme knowledge. For + searching messages there is the mu:store:for-each function, which takes two + arguments; the first argument is a function that will be called for each + message found. The optional second argument is the search expression (following + 'mu find' syntax); if don't provide the argument, all messages match. + + So how does this work in practice? Let's see I want to see the subject and + sender for messages about milk: + + ,---- + | (mu:store:for-each (lambda(msg) (format #t "~s ~s\n" (mu:msg:from msg) (mu:msg:subject msg))) "milk") + `---- + + or slightly more readable: + + ,---- + | (mu:store:for-each + | (lambda(msg) + | (format #t "~s ~s\n" (mu:msg:from msg) (mu:msg:subject msg))) + | "milk") + `---- + + As you can see, I provide an anonymous ('lambda') function which will be + called for each message matching 'milk'. Admittedly, this requires a bit of + Scheme-knowledge... but this time is good as any to learn this nice + language. + + +** Can I do some statistics on my messages? + + Yes you can. In fact, it's pretty easy. If you load (in the muile/ directory) + the file 'mu-stats.scm': + + ,---- + | (load "mu-stats.scm") + `---- + + you'll get a bunch of functions (with names starting with 'mu:stats') to make + this very easy. Let's see, suppose I want to see how many messages I get per + weekday: + + ,---- + | scheme@(guile-user)> (mu:stats:per-weekday) + | $1 = ((0 . 2255) (1 . 2788) (2 . 2868) (3 . 2599) (4 . 2629) (5 . 2287) (6 . 1851)) + `---- + + Note, Sunday=0, Monday=1 and so on. Apparently, I get/send most of e-mail on + Tuesdays, and least on Saturday. + + And note that mu:stats:per-weekdays takes an optional search expression + argument, to limit the results to messages matching that, e.g., to only + consider messages related to emacs during this year: + + ,---- + | scheme@(guile-user)> (mu:stats:per-weekday "emacs date:2011..now") + | $8 = ((0 . 54) (1 . 22) (2 . 46) (3 . 47) (4 . 39) (5 . 54) (6 . 50)) + `---- + + There's also 'mu:stats:per-month', 'mu:stats:per-year', 'mu:stats:per-hour'. + I learnt that during 3-4am I sent/receive only about a third of what I sent + during 11-12pm. + +** What about getting the top-10 people in the To:-field? + + Easy. + + ,---- + | scheme@(guile-user)> (mu:stats:top-n-to) + | $1 = ((("Abc" "myself@example.com") . 4465) (("Def" "somebodyelse@example.com") . 2114) + | (and so on) + `---- + + I've changed the names a bit to protect the innocent, but what the function + does is return a list of pairs of + + (<name> <email>) . <frequency> + + descending in order of frequency. Note, 'mu:stats:top-n-to' takes two + optional arguments - first the 'n' in top-n (default is 10), and seconds as + search expression to limit the messages considered. + + There are also the functions 'mu:stats:top-n-subject' and + 'mu:stats:top-n-from' which do the same, mutatis mutandis, and it's quite + easy to add your own (see the mu-stats.scm for examples) + +** What about showing the results in a table? + + Even easier. Try: + + ,---- + | (mu:stats:table (mu:stats:top-n-to)) + `---- + + or + + ,---- + | (mu:stats:table (mu:stats:per-weekday)) + `---- + + You can also export the table: + + ,---- + | (mu:stats:export (mu:stats:per-weekday)) + `---- + + which will create a temporary file with the results, for further processing + in e.g. 'R' or 'gnuplot'. + + +[1] http://www.gnu.org/s/guile/ +[2] http://en.wikipedia.org/wiki/Scheme_(programming_language) + +# Local Variables: +# mode: org; org-startup-folded: nil +# End: diff --git a/guile/mu/contact.scm b/guile/mu/contact.scm new file mode 100644 index 0000000..843d9c4 --- /dev/null +++ b/guile/mu/contact.scm @@ -0,0 +1,4 @@ +(define-module (mu contact) :use-module(mu)) +(display "(mu contact) is deprecated, please remove from (use-modules ...)") +(newline) + diff --git a/guile/mu/message.scm b/guile/mu/message.scm new file mode 100644 index 0000000..bc9b27a --- /dev/null +++ b/guile/mu/message.scm @@ -0,0 +1,4 @@ +(define-module (mu message) :use-module (mu)) +(display "(mu message) is deprecated, please remove from (use-modules ...)") +(newline) + diff --git a/guile/mu/part.scm b/guile/mu/part.scm new file mode 100644 index 0000000..f9b9cd3 --- /dev/null +++ b/guile/mu/part.scm @@ -0,0 +1,4 @@ +(define-module (mu part) :use-module (mu)) +(display "(mu part) is deprecated, please remove from (use-modules ...)") +(newline) + diff --git a/guile/mu/plot.scm b/guile/mu/plot.scm new file mode 100644 index 0000000..cd09e22 --- /dev/null +++ b/guile/mu/plot.scm @@ -0,0 +1,83 @@ +;; +;; Copyright (C) 2011-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +(define-module (mu plot) + :use-module (mu) + :use-module (ice-9 popen) + :export ( mu:plot ;; alias for mu:plot-histogram + mu:plot-histogram + )) + +(define (export-pairs pairs) + "Write a temporary file with the list of PAIRS in table format, and +return the file name." + (let* ((output (mkstemp "/tmp/mu-guile-XXXXXX" "w")) + (datafile (port-filename output))) + (for-each + (lambda(pair) + (display (format #f "~a ~a\n" (car pair) (cdr pair)) output)) + pairs) + (close output) + datafile)) + +(define (find-program-in-path prog) + "Find exutable program PROG in PATH; return the full path, or #f if +not found." + (let* ((path (parse-path (getenv "PATH"))) + (progpath (search-path path prog))) + (if (not progpath) + #f + (if (access? progpath X_OK) ;; is + progpath + #f)))) + +(define* (mu:plot-histogram data title x-label y-label + #:optional (output "dumb") (extra-gnuplot-opts '())) + "Plot DATA with TITLE, X-LABEL and X-LABEL using the gnuplot +program. DATA is a list of cons-pairs (X . Y). + + OUTPUT is a string +that determines the type of output that gnuplot produces, depending on +the system. Which options are available depends on the particulars for +the gnuplot installation, but typical examples would be \"dumb\" for +text-only display, \"wxterm\" to write to a graphical window, or +\"png\" to write a PNG-image to stdout. + +EXTRA-GNUPLOT-OPTS is a list +of any additional options for gnuplot." + (if (not (find-program-in-path "gnuplot")) + (error "cannot find 'gnuplot' in path")) + (when (zero? (length data)) + (error "No data for plotting")) + (let* ((datafile (export-pairs data)) + (gnuplot (open-pipe "gnuplot -p" OPEN_WRITE)) + (recipe + (string-append + "reset\n" + "set term " (or output "dumb") "\n" + "set title \"" title "\"\n" + "set xlabel \"" x-label "\"\n" + "set ylabel \"" y-label "\"\n" + "set boxwidth 0.9\n" + (string-join extra-gnuplot-opts "\n") + "plot \"" datafile "\" using 2:xticlabels(1) with boxes fs solid title \"\"\n"))) + (display recipe gnuplot) + (close-pipe gnuplot))) + +;; backward compatibility +(define mu:plot mu:plot-histogram) diff --git a/guile/mu/script.scm b/guile/mu/script.scm new file mode 100644 index 0000000..45aad8a --- /dev/null +++ b/guile/mu/script.scm @@ -0,0 +1,58 @@ +;; Copyright (C) 2012-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +(define-module (mu script) + :export (mu:run-stats)) + +(use-modules (ice-9 getopt-long) (ice-9 optargs) (ice-9 popen) (ice-9 format)) +(use-modules (mu) (mu stats) (mu plot)) + +(define (help-and-exit) + "Show some help." + (display + (string-append "usage: script [--help] [--textonly] " + "[--muhome=<muhome>] [--query=<query>") + (newline)) + (exit 0)) + +(define (mu:run-stats args func) + "Run some statistics function. +Interpret argument-list ARGS (like command-line +arguments). Possible arguments are: + --help (show some help and exit) + --muhome (path to alternative mu home directory) + --output (a string describing the output, e.g. \"dumb\", \"png\" \"wxt\") + searchexpr (a search query) +then call FUNC with args SEARCHEXPR and OUTPUT." + (setlocale LC_ALL "") + (let* ((optionspec '((muhome (value #t)) + (query (value #t)) + (output (value #f)) + (time-unit (value #t)) ;; Ignore. + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (query (option-ref options 'query #f)) + (help (option-ref options 'help #f)) + (output (option-ref options 'output #f)) + (muhome (option-ref options 'muhome #f)) + (restargs (option-ref options '() #f))) + (if help (help-and-exit)) + (mu:initialize muhome) + (func (or query "") output))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/mu/stats.scm b/guile/mu/stats.scm new file mode 100644 index 0000000..1e73605 --- /dev/null +++ b/guile/mu/stats.scm @@ -0,0 +1,167 @@ +;; +;; Copyright (C) 2011-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +(define-module (mu stats) + :use-module (oop goops) + :use-module (mu) + :use-module (srfi srfi-1) + :use-module (ice-9 i18n) + :use-module (ice-9 r5rs) + :export ( mu:tabulate + mu:top-n-most-frequent + mu:count + mu:average + mu:stddev + mu:correl + mu:max + mu:min + mu:weekday-numbers->names + mu:month-numbers->names)) + + +(define* (mu:tabulate func #:optional (expr #t)) + "Execute FUNC for each message matching EXPR, and return an alist +with maps each result of FUNC to its frequency. If the result of FUNC +is a list, add each of its values separately. + FUNC is a function takes a <mu-message> instance as its argument. For +example, to tabulate messages by weekday, one could use: + (mu:tabulate (lambda(msg) (tm:wday (localtime (date msg))))), and +get back a list like + ((1 . 2) (2 . 5)(3 . 4)(4 . 4)(5 . 12)(6 . 7)(7. 2))." + (let* ((table '()) + ;; func to add a value to our table + (update-table + (lambda (val) + (let ((old-freq (or (assoc-ref table val) 0))) + (set! table (assoc-set! table val (1+ old-freq))))))) + (mu:for-each-message + (lambda(msg) + (let ((val (func msg))) + (if (list? val) + (for-each update-table val) + (update-table val)))) + expr) + table)) + +(define* (top-n func less n #:optional (expr #t)) + "Take the results of (mu:tabulate FUNC EXPR), sort using LESS (a +function taking two arguments A and B (cons cells, (VAL . FREQ)), and +returns #t if A < B, #f otherwise), and then take the first N." + (take (sort (mu:tabulate func expr) less) n)) + +(define* (mu:top-n-most-frequent func n #:optional (expr #t)) + "Take the results of (mu:tabulate FUNC EXPR), and return the N items +with the highest frequency." + (top-n func (lambda (a b) (> (cdr a) (cdr b))) n expr)) + +(define* (mu:count #:optional (expr #t)) + "Count the number of messages matching EXPR. If EXPR is not +provided, match /all/ messages." + (let ((num 0)) + (mu:for-each-message + (lambda (msg) (set! num (1+ num))) + expr) + num)) + +(define (average lst) + "Calculate the average of a list LST of numbers, or #f if undefined." + (if (null? lst) + #f + (/ (apply + lst) (length lst)))) + +(define (stddev lst) + "Calculate the standard deviation of a list LST of numbers or #f if +undefined." + (let* ((avg (average lst)) + (sosq (if avg + (apply + (map (lambda (x)(* (- x avg) (- x avg))) lst))))) + (if sosq + (sqrt (/ sosq (length lst)))))) + + +(define* (mu:average func #:optional (expr #t)) + "Get the average value of FUNC applied to all messages matching +EXPR (or #t for all). Returns #f if undefined." + (average (map func (mu:message-list expr)))) + +(define* (mu:stddev func #:optional (expr #t)) + "Get the standard deviation the the values of FUNC applied to all +messages matching EXPR (or #t for all). This is the 'population' stddev, +not the 'sample' stddev. Returns #f if undefined." + (stddev (map func (mu:message-list expr)))) + +(define* (mu:max func #:optional (expr #t)) + "Get the maximum value of FUNC applied to all messages matching +EXPR (or #t for all). Returns #f if undefined." + (apply max (map func (mu:message-list expr)))) + +(define* (mu:min func #:optional (expr #t)) + "Get the minimum value of FUNC applied to all messages matching +EXPR (or #t for all). Returns #f if undefined." + (apply min (map func (mu:message-list expr)))) + + +(define (correl lst) + "Calculate Pearson's correlation coefficient for a list LST of cons +pair, where the car and cdr of the pairs are values from data set 1 +and 2, respectively." + (let ((n (length lst)) + (sx (apply + (map car lst))) + (sy (apply + (map cdr lst))) + (sxy (apply + (map (lambda (cell) (* (car cell) (cdr cell))) lst))) + (sxx (apply + (map (lambda (cell) (* (car cell) (car cell))) lst))) + (syy (apply + (map (lambda (cell) (* (cdr cell) (cdr cell))) lst)))) + (/ (- (* n sxy) (* sx sy)) + (sqrt (* (- (* n sxx) (* sx sx)) (- (* n syy) (* sy sy))))))) + +(define* (mu:correl func1 func2 #:optional (expr #t)) + "Determine Pearson's correlation coefficient between the value for +functions FUNC1 and FUNC2 to all messages matching EXPR (or #t for +all). Returns #f if undefined." + (let ((data + (map (lambda (msg) + (cons (func1 msg) (func2 msg))) + (mu:message-list expr)))) + (if data (correl data) #f))) + + +;; a list of abbreviated, localized day names +(define day-names + (map locale-day-short (iota 7 1))) + +(define (mu:weekday-numbers->names table) + "Convert a list of pairs with the car denoting a day number (0-6) +into a list of pairs with the car replaced by the corresponding day +name (abbreviated) for the current locale." + (map + (lambda (pair) + (cons (list-ref day-names (car pair)) (cdr pair))) + table)) + +;; a list of abbreviated, localized month names +(define month-names + (map locale-month-short (iota 12 1))) + +(define (mu:month-numbers->names table) + "Convert a list of pairs with the car denoting a month number (0-11) +into a list of pairs with the car replaced by the corresponding day +name (abbreviated)." + (map + (lambda (pair) + (cons (list-ref month-names (car pair)) (cdr pair))) + table)) diff --git a/guile/scripts/find-dups.scm b/guile/scripts/find-dups.scm new file mode 100755 index 0000000..c4b6263 --- /dev/null +++ b/guile/scripts/find-dups.scm @@ -0,0 +1,119 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# +;; +;; Copyright (C) 2013-2015 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +;; INFO: Find duplicate messages +;; INFO: options: +;; INFO: --muhome=<muhome>: path to mu home dir +;; INFO: --delete: delete all but the first one (experimental, be careful!) + +(use-modules (mu) (mu script) (mu stats)) +(use-modules (ice-9 getopt-long) (ice-9 optargs) + (ice-9 popen) (ice-9 format) (ice-9 rdelim)) + +(define (md5sum path) + (let* ((port (open-pipe* OPEN_READ "md5sum" path)) + (md5 (read-delimited " " port))) + (close-pipe port) + md5)) + +(define (find-dups delete expr) + (let ((id-table (make-hash-table 20000))) + ;; fill the hash with <msgid-size> => <list of paths> + (mu:for-each-message + (lambda (msg) + (let* ((id (format #f "~a-~d" (mu:message-id msg) + (mu:size msg))) + (lst (hash-ref id-table id))) + (if lst + (set! lst (cons (mu:path msg) lst)) + (set! lst (list (mu:path msg)))) + (hash-set! id-table id lst))) + expr) + ;; list all the paths with multiple elements; check the md5sum to + ;; make 100%-minus-ε sure they are really the same file. + (hash-for-each + (lambda (id paths) + (if (> (length paths) 1) + (let ((hash (make-hash-table 10))) + (for-each + (lambda (path) + (when (file-exists? path) + (let* ((md5 (md5sum path)) (lst (hash-ref hash md5))) + (if lst + (set! lst (cons path lst)) + (set! lst (list path))) + (hash-set! hash md5 lst)))) + paths) + ;; hash now maps the md5sum to the messages... + (hash-for-each + (lambda (md5 mpaths) + (if (> (length mpaths) 1) + (begin + ;;(format #t "md5sum: ~a:\n" md5) + (let ((num 1)) + (for-each + (lambda (path) + (if (equal? num 1) + (format #t "~a\n" path) + (begin + (format #t "~a: ~a\n" (if delete "deleting" "dup") path) + (if delete (delete-file path)))) + (set! num (+ 1 num))) + mpaths))))) + hash)))) + id-table))) + + + +(define (main args) + "Find duplicate messages and, potentially, delete the dups. + Be careful with that! +Interpret argument-list ARGS (like command-line +arguments). Possible arguments are: + --muhome (path to alternative mu home directory). + --delete (delete all but the first one). Run mu index afterwards. + --expr (expression to constrain search)." + (setlocale LC_ALL "") + (let* ((optionspec '( (muhome (value #t)) + (delete (value #f)) + (expr (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (help (option-ref options 'help #f)) + (delete (option-ref options 'delete #f)) + (expr (option-ref options 'expr #t)) + (muhome (option-ref options 'muhome #f))) + (mu:initialize muhome) + (find-dups delete expr))) + + +;; Local Variables: +;; mode: scheme +;; End: + + + + + + + + + diff --git a/guile/scripts/histogram.scm b/guile/scripts/histogram.scm new file mode 100755 index 0000000..b845f28 --- /dev/null +++ b/guile/scripts/histogram.scm @@ -0,0 +1,127 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# +;; Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. + +;; INFO: Histogram of the number of messages per time-unit +;; INFO: Options: +;; INFO: --query=<query>: limit to messages matching query +;; INFO: --muhome=<muhome>: path to mu home dir +;; INFO: --time-unit: hour|day|month|year|month-year +;; INFO: --output: the output format, such as "png", "wxt" +;; INFO: (depending on the environment) + +(use-modules (mu) (mu stats) (mu plot) + (ice-9 getopt-long) (ice-9 format)) + +(define (per-hour expr output) + "Count the total number of messages per hour that match EXPR. +OUTPUT corresponds to the output format, as per gnuplot's 'set terminal'." + (mu:plot-histogram + (sort + (mu:tabulate + (lambda (msg) + (tm:hour (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per hour matching ~a" expr) + "Hour" "Messages" output)) + + +(define (per-day expr output) + "Count the total number of messages for each weekday (0-6 for +Sun..Sat) that match EXPR. OUTPUT corresponds to the output format, as +per gnuplot's 'set terminal'." + (mu:plot-histogram + (mu:weekday-numbers->names + (sort (mu:tabulate + (lambda (msg) + (tm:wday (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y))))) + (format #f "Messages per weekday matching ~a" expr) + "Day" "Messages" output)) + +(define (per-month expr output) + "Count the total number of messages per month that match EXPR. +OUTPUT corresponds to the output format, as per gnuplot's 'set terminal'." + (mu:plot-histogram + (mu:month-numbers->names + (sort + (mu:tabulate + (lambda (msg) + (tm:mon (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y))))) + (format #f "Messages per month matching ~a" expr) + "Month" "Messages" output)) + +(define (per-year expr output) + "Count the total number of messages per year that match EXPR. OUTPUT corresponds +to the output format, as per gnuplot's 'set terminal'." + (mu:plot-histogram + (sort (mu:tabulate + (lambda (msg) + (+ 1900 (tm:year (localtime (mu:date msg))))) expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per year matching ~a" expr) + "Year" "Messages" output)) + + +(define (per-year-month expr output) + "Count the total number of messages for each year and month that match EXPR. +OUTPUT corresponds to the output format, as per gnuplot's 'set terminal'." + (mu:plot-histogram + (sort (mu:tabulate + (lambda (msg) + (string->number + (format #f "~d~2'0d" + (+ 1900 (tm:year (localtime (mu:date msg)))) + (tm:mon (localtime (mu:date msg)))))) + expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per year/month matching ~a" expr) + "Year/Month" "Messages" output)) + +(define (main args) + (let* ((optionspec + '((time-unit (value #t)) + (query (value #t)) + (muhome (value #t)) + (output (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (help (option-ref options 'help #f)) + (time-unit (option-ref options 'time-unit "year")) + (muhome (option-ref options 'muhome #f)) + (query (option-ref options 'query "")) + (output (option-ref options 'output "dumb")) + (rest (option-ref options '() #f)) + (func + (cond + ((equal? time-unit "hour") per-hour) + ((equal? time-unit "day") per-day) + ((equal? time-unit "month") per-month) + ((equal? time-unit "year") per-year) + ((equal? time-unit "year-month") per-year-month) + (else #f)))) + (setlocale LC_ALL "") + (unless func + (display "error: unknown time-unit\n") + (set! help #t)) + (if help + (begin + (display + (string-append "parameters: [--help] [--output=dumb|png|wxt] " + "[--muhome=<muhome>] [--query=<query>]" + "[--time-unit=hour|day|month|year|year-month]")) + (newline)) + (begin + (mu:initialize muhome) + (func query output))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/scripts/msgs-count.scm b/guile/scripts/msgs-count.scm new file mode 100755 index 0000000..9a73efe --- /dev/null +++ b/guile/scripts/msgs-count.scm @@ -0,0 +1,40 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# +;; Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +;; INFO: Count the number of messages matching some query +;; INFO: options: +;; INFO: --query=<query>: limit to messages matching query +;; INFO: --muhome=<muhome>: path to mu home dir (optional) + +(use-modules (mu) (mu script) (mu stats)) + +(define (count expr output) + "Print the total number of messages matching the query EXPR. +OUTPUT is ignored." + (display (mu:count expr)) + (newline)) + +(define (main args) + (mu:run-stats args count)) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/tests/meson.build b/guile/tests/meson.build new file mode 100644 index 0000000..07b3790 --- /dev/null +++ b/guile/tests/meson.build @@ -0,0 +1,38 @@ +## Copyright (C) 2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# guile test; they don't work with ASAN. +# +if get_option('b_sanitize') == 'none' + guile_load_path = join_paths(meson.project_source_root(), 'guile') + guile_extensions_path = ':'.join([ + join_paths(meson.project_build_root(), 'guile'), + meson.current_build_dir()]) + + test('test-mu-guile', + executable('test-mu-guile', + 'test-mu-guile.cc', + install: false, + cpp_args: [ + '-DABS_SRCDIR="' + meson.current_source_dir() + '"', + '-DGUILE_LOAD_PATH="' + guile_load_path + '"', + '-DGUILE_EXTENSIONS_PATH="' + guile_extensions_path + '"' + ], + dependencies: [glib_dep, lib_mu_dep])) +else + message('sanitizer build; skip guile test') +endif diff --git a/guile/tests/test-mu-guile.cc b/guile/tests/test-mu-guile.cc new file mode 100644 index 0000000..09a53d0 --- /dev/null +++ b/guile/tests/test-mu-guile.cc @@ -0,0 +1,130 @@ +/* +** Copyright (C) 2012-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <glib.h> +#include <glib/gstdio.h> + +#include <lib/mu-query.hh> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +#include "utils/mu-test-utils.hh" +#include <lib/mu-store.hh> +#include <utils/mu-utils.hh> + +using namespace Mu; + +static std::string test_dir; + +static std::string +fill_database(void) +{ + const auto cmdline = mu_format( + "/bin/sh -c '" + "{} init --muhome={} --maildir={} --quiet; " + "{} index --muhome={} --quiet'", + MU_PROGRAM, + test_dir, + MU_TESTMAILDIR2, + MU_PROGRAM, + test_dir); + + if (g_test_verbose()) + mu_println("{}", cmdline); + + GError *err{}; + if (!g_spawn_command_line_sync(cmdline.c_str(), NULL, NULL, NULL, &err)) { + mu_printerrln("Error: {}", err ? err->message : "?"); + g_clear_error(&err); + g_assert(0); + } + + return test_dir; +} + +static void +test_something(const char* what) +{ + g_setenv("GUILE_AUTO_COMPILE", "0", TRUE); + g_setenv("GUILE_LOAD_PATH", GUILE_LOAD_PATH, TRUE); + g_setenv("GUILE_EXTENSIONS_PATH",GUILE_EXTENSIONS_PATH, TRUE); + + if (g_test_verbose()) + g_print("GUILE_LOAD_PATH: %s\n", GUILE_LOAD_PATH); + + const auto dir = fill_database(); + const auto cmdline = mu_format("{} -q -e main {}/test-mu-guile.scm " + "--muhome={} --test={}", + GUILE_BINARY, ABS_SRCDIR, + dir, what); + + if (g_test_verbose()) + mu_println("cmdline: {}", cmdline); + + GError *err{}; + int status{}; + if (!g_spawn_command_line_sync(cmdline.c_str(), NULL, NULL, &status, &err) || + status != 0) { + mu_printerrln("Error: {}", err ? err->message : "something went wrong"); + g_clear_error(&err); + g_assert(0); + } +} + +static void +test_mu_guile_queries(void) +{ + test_something("queries"); +} + +static void +test_mu_guile_messages(void) +{ + test_something("message"); +} + +static void +test_mu_guile_stats(void) +{ + test_something("stats"); +} + +int +main(int argc, char* argv[]) +{ + int rv; + TempDir tempdir; + test_dir = tempdir.path(); + + mu_test_init(&argc, &argv); + + if (!set_en_us_utf8_locale()) + return 0; /* don't error out... */ + + g_test_add_func("/guile/queries", test_mu_guile_queries); + g_test_add_func("/guile/message", test_mu_guile_messages); + g_test_add_func("/guile/stats", test_mu_guile_stats); + + rv = g_test_run(); + + return rv; +} diff --git a/guile/tests/test-mu-guile.scm b/guile/tests/test-mu-guile.scm new file mode 100755 index 0000000..afa4f48 --- /dev/null +++ b/guile/tests/test-mu-guile.scm @@ -0,0 +1,124 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; Copyright (C) 2012-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +(setlocale LC_ALL "") + +(use-modules (srfi srfi-1)) +(use-modules (ice-9 getopt-long) (ice-9 optargs) (ice-9 popen) (ice-9 format)) +(use-modules (mu) (mu stats)) + +(define (n-results-or-exit query n) + "Run QUERY, and exit 1 if the number of results != N." + (let ((lst (mu:message-list query))) + (if (not (= (length lst) n)) + (begin + (simple-format (current-error-port) "Query: \"~A\"; expected ~A, got ~A\n" + query n (length lst)) + (exit 1))))) + +(define (test-queries) + "Test a bunch of queries (or die trying)." + (n-results-or-exit "hello" 1) + (n-results-or-exit "f:john fruit" 1) + (n-results-or-exit "f:soc@example.com" 1) + (n-results-or-exit "t:alki@example.com" 1) + (n-results-or-exit "t:alcibiades" 1) + (n-results-or-exit "f:soc@example.com OR f:john" 2) + (n-results-or-exit "f:soc@example.com OR f:john OR t:edmond" 3) + (n-results-or-exit "t:julius" 1) + (n-results-or-exit "s:dude" 1) + (n-results-or-exit "t:dantès" 1) + (n-results-or-exit "file:sittingbull.jpg" 1) + (n-results-or-exit "file:custer.jpg" 1) + (n-results-or-exit "file:custer.*" 1) + (n-results-or-exit "j:sit*" 1) + (n-results-or-exit "mime:image/jpeg" 1) + (n-results-or-exit "mime:text/plain" 14) + (n-results-or-exit "y:text*" 14) + (n-results-or-exit "y:image*" 1) + (n-results-or-exit "mime:message/rfc822" 2)) + +(define (error-exit msg . args) + "Print error and exit." + (let ((msg (apply format #f msg args))) + (simple-format (current-error-port) "*ERROR*: ~A\n" msg) + (exit 1))) + +(define (str-equal-or-exit got exp) + "S1 == S2 or exit 1." + ;; (format #t "'~A' <=> '~A'\n" s1 s2) + (if (not (string= exp got)) + (error-exit "Expected \"~A\", got \"~A\"\n" exp got))) + +(define (test-message) + "Test functions for a particular message." + + (let ((msg (car (mu:message-list "hello")))) + (str-equal-or-exit (mu:subject msg) "Fwd: rfc822") + (str-equal-or-exit (mu:to msg) "martin") + (str-equal-or-exit (mu:from msg) "foobar <foo@example.com>") + (str-equal-or-exit (mu:header msg "X-Mailer") "Ximian Evolution 1.4.5") + + (if (not (equal? (mu:priority msg) mu:prio:normal)) + (error-exit "Expected ~A, got ~A" (mu:priority msg) mu:prio:normal))) + + (let ((msg (car (mu:message-list "atoms")))) + (str-equal-or-exit (mu:subject msg) "atoms") + (str-equal-or-exit (mu:to msg) "Democritus <demo@example.com>") + (str-equal-or-exit (mu:from msg) "Richard P. Feynman <rpf@example.com>") + ;;(str-equal-or-exit (mu:header msg "Content-transfer-encoding") "7BIT") + + (if (not (equal? (mu:priority msg) mu:prio:high)) + (error-exit "Expected ~a, got ~a" (mu:priority msg) mu:prio:high)))) + +(define (num-equal-or-exit got exp) + "S1 == S2 or exit 1." + ;; (format #t "'~A' <=> '~A'\n" s1 s2) + (if (not (= exp got)) + (error-exit "Expected \"~S\", got \"~S\"\n" exp got))) + +(define (test-stats) + "Test statistical functions." + ;; average + (num-equal-or-exit (mu:average mu:size) 82601/14) + (num-equal-or-exit (floor (mu:stddev mu:size)) 12637.0) + (num-equal-or-exit (mu:max mu:size) 46308) + (num-equal-or-exit (mu:min mu:size) 111)) + +(define (main args) + (let* ((optionspec '((muhome (value #t)) + (test (value #t)))) + (options (getopt-long args optionspec)) + (muhome (option-ref options 'muhome #f)) + (test (option-ref options 'test #f))) + + (mu:initialize muhome) + + (if test + (cond + ((string= test "queries") (test-queries)) + ((string= test "message") (test-message)) + ((string= test "stats") (test-stats)) + (#t (exit 1)))))) + + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/lib/meson.build b/lib/meson.build new file mode 100644 index 0000000..b3b519d --- /dev/null +++ b/lib/meson.build @@ -0,0 +1,95 @@ +## Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +subdir('utils') +subdir('message') + +lib_mu=static_library( + 'mu', + [ + # db + 'mu-config.cc', + 'mu-contacts-cache.cc', + 'mu-store.cc', + 'mu-xapian-db.cc', + # querying + 'mu-query-macros.cc', + 'mu-query-match-deciders.cc', + 'mu-query-parser.cc', + 'mu-query-processor.cc', + 'mu-query-threads.cc', + 'mu-query-xapianizer.cc', + 'mu-query.cc', + # indexing + 'mu-indexer.cc', + 'mu-scanner.cc', + # mu4e + 'mu-server.cc', + # misc + 'mu-maildir.cc', + 'mu-script.cc', + ], + dependencies: [ + glib_dep, + gio_dep, + gmime_dep, + xapian_dep, + guile_dep, + config_h_dep, + lib_mu_utils_dep, + lib_mu_message_dep], + install: false) + +lib_mu_dep = declare_dependency( + link_with: lib_mu, + dependencies: [ lib_mu_message_dep, thread_dep ], + include_directories: + include_directories(['.', '..'])) + +# +# dev helpers +# + +process_query = executable('process-query', [ 'mu-query-processor.cc'], + install: false, + cpp_args: ['-DBUILD_PROCESS_QUERY'], + dependencies: [glib_dep, lib_mu_dep]) + +parse_query = executable( 'parse-query', [ 'mu-query-parser.cc' ], + install: false, + cpp_args: ['-DBUILD_PARSE_QUERY'], + dependencies: [glib_dep, lib_mu_dep]) + +parse_query_expand = executable( 'parse-query-expand', [ 'mu-query-parser.cc' ], + install: false, + cpp_args: ['-DBUILD_PARSE_QUERY_EXPAND'], + dependencies: [glib_dep, lib_mu_dep]) + +xapian_query = executable('xapianize-query', [ 'mu-query-xapianizer.cc' ], + install: false, + cpp_args: ['-DBUILD_XAPIANIZE_QUERY'], + dependencies: [glib_dep, lib_mu_dep]) + +list_maildirs = executable('list-maildirs', 'mu-scanner.cc', + install: false, + cpp_args: ['-DBUILD_LIST_MAILDIRS'], + dependencies: [glib_dep, config_h_dep, + lib_mu_utils_dep]) + +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/lib/message/meson.build b/lib/message/meson.build new file mode 100644 index 0000000..006bb18 --- /dev/null +++ b/lib/message/meson.build @@ -0,0 +1,47 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +lib_mu_message=static_library( + 'mu-message', + [ + 'mu-message.cc', + 'mu-message-file.cc', + 'mu-message-part.cc', + 'mu-contact.cc', + 'mu-document.cc', + 'mu-fields.cc', + 'mu-flags.cc', + 'mu-priority.cc', + 'mu-mime-object.cc', + ], + dependencies: [ + glib_dep, + gmime_dep, + xapian_dep, + config_h_dep, + lib_mu_utils_dep], + install: false) + +lib_mu_message_dep = declare_dependency( + link_with: lib_mu_message, + dependencies: [ xapian_dep, gmime_dep, lib_mu_utils_dep, config_h_dep ], + include_directories: + include_directories(['.', '..'])) + +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/lib/message/mu-contact.cc b/lib/message/mu-contact.cc new file mode 100644 index 0000000..c6439b0 --- /dev/null +++ b/lib/message/mu-contact.cc @@ -0,0 +1,207 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-contact.hh" +#include "mu-message.hh" +#include "utils/mu-utils.hh" +#include "mu-mime-object.hh" + +#include <gmime/gmime.h> +#include <glib.h> +#include <string> + +using namespace Mu; + +std::string +Contact::display_name() const +{ + const auto needs_quoting= [](const std::string& n) { + for (auto& c: n) + if (c == ',' || c == '"' || c == '@') + return true; + return false; + }; + + if (name.empty()) + return email; + else if (!needs_quoting(name)) + return name + " <" + email + '>'; + else + return Mu::quote(name) + " <" + email + '>'; +} + +std::string +Mu::to_string(const Mu::Contacts& contacts) +{ + std::string res; + + seq_for_each(contacts, [&](auto&& contact) { + if (res.empty()) + res = contact.display_name(); + else + res += ", " + contact.display_name(); + }); + + return res; +} + +size_t +Mu::lowercase_hash(const std::string& s) +{ + std::size_t djb = 5381; // djb hash + for (const auto c : s) + djb = ((djb << 5) + djb) + + static_cast<size_t>(g_ascii_tolower(c)); + return djb; +} + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_ctor_foo() +{ + Contact c{ + "foo@example.com", + "Foo Bar", + Contact::Type::Bcc, + 1645214647 + }; + + assert_equal(c.email, "foo@example.com"); + assert_equal(c.name, "Foo Bar"); + g_assert_true(*c.field_id() == Field::Id::Bcc); + g_assert_cmpuint(c.message_date,==,1645214647); + + assert_equal(c.display_name(), "Foo Bar <foo@example.com>"); +} + + +static void +test_ctor_blinky() +{ + Contact c{ + "bar@example.com", + "Blinky", + 1645215014, + true, /* personal */ + 13, /*freq*/ + 12345 /* tstamp */ + }; + + assert_equal(c.email, "bar@example.com"); + assert_equal(c.name, "Blinky"); + g_assert_true(c.personal); + g_assert_cmpuint(c.frequency,==,13); + g_assert_cmpuint(c.tstamp,==,12345); + g_assert_cmpuint(c.message_date,==,1645215014); + + assert_equal(c.display_name(), "Blinky <bar@example.com>"); +} + +static void +test_ctor_cleanup() +{ + Contact c{ + "bar@example.com", + "Bli\nky", + 1645215014, + true, /* personal */ + 13, /*freq*/ + 12345 /* tstamp */ + }; + + assert_equal(c.email, "bar@example.com"); + assert_equal(c.name, "Bli ky"); + g_assert_true(c.personal); + g_assert_cmpuint(c.frequency,==,13); + g_assert_cmpuint(c.tstamp,==,12345); + g_assert_cmpuint(c.message_date,==,1645215014); + + assert_equal(c.display_name(), "Bli ky <bar@example.com>"); +} + +static void +test_encode() +{ + Contact c{ + "cassius@example.com", + "Ali, Muhammad \"The Greatest\"", + 345, + false, /* personal */ + 333, /*freq*/ + 768 /* tstamp */ + }; + + assert_equal(c.email, "cassius@example.com"); + assert_equal(c.name, "Ali, Muhammad \"The Greatest\""); + g_assert_false(c.personal); + g_assert_cmpuint(c.frequency,==,333); + g_assert_cmpuint(c.tstamp,==,768); + g_assert_cmpuint(c.message_date,==,345); + + assert_equal(c.display_name(), + "\"Ali, Muhammad \\\"The Greatest\\\"\" <cassius@example.com>"); +} + + +static void +test_sender() +{ + Contact c{"aa@example.com", "Anders Ångström", + Contact::Type::Sender, 54321}; + + assert_equal(c.email, "aa@example.com"); + assert_equal(c.name, "Anders Ångström"); + g_assert_false(c.personal); + g_assert_cmpuint(c.frequency,==,1); + g_assert_cmpuint(c.message_date,==,54321); + + g_assert_false(!!c.field_id()); +} + + +static void +test_misc() +{ + g_assert_false(!!contact_type_from_field_id(Field::Id::Subject)); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + g_mime_init(); + + g_test_add_func("/message/contact/ctor-foo", test_ctor_foo); + g_test_add_func("/message/contact/ctor-blinky", test_ctor_blinky); + g_test_add_func("/message/contact/ctor-cleanup", test_ctor_cleanup); + g_test_add_func("/message/contact/encode", test_encode); + + g_test_add_func("/message/contact/sender", test_sender); + g_test_add_func("/message/contact/misc", test_misc); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-contact.hh b/lib/message/mu-contact.hh new file mode 100644 index 0000000..d417d4e --- /dev/null +++ b/lib/message/mu-contact.hh @@ -0,0 +1,219 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MESSAGE_CONTACT_HH__ +#define MU_MESSAGE_CONTACT_HH__ + +#include <functional> +#include <string> +#include <vector> +#include <functional> +#include <cctype> +#include <cstring> +#include <cstdlib> +#include <ctime> + +#include <utils/mu-option.hh> +#include "mu-fields.hh" + +struct _InternetAddressList; + +namespace Mu { + +/** + * Get the hash value for a lowercase value of s; useful for email-addresses + * + * @param s a string + * + * @return a hash value. + */ +size_t lowercase_hash(const std::string& s); + +struct Contact { + enum struct Type { + None, Sender, From, ReplyTo, To, Cc, Bcc + }; + + /** + * Construct a new Contact + * + * @param email_ email address + * @param name_ name or empty + * @param type_ contact field type + * @param message_date_ data for the message for this contact + */ + Contact(const std::string& email_, const std::string& name_ = "", + Type type_ = Type::None, ::time_t message_date_ = 0) + : email{email_}, name{name_}, type{type_}, + message_date{message_date_}, personal{}, frequency{1}, tstamp{} + { cleanup_name(); } + + /** + * Construct a new Contact + * + * @param email_ email address + * @param name_ name or empty + * @param message_date_ date of message this contact originate from + * @param personal_ is this a personal contact? + * @param freq_ how often was this contact seen? + * @param tstamp_ timestamp for last change + */ + Contact(const std::string& email_, const std::string& name_, + time_t message_date_, bool personal_, size_t freq_, + int64_t tstamp_) + : email{email_}, name{name_}, type{Type::None}, + message_date{message_date_}, personal{personal_}, frequency{freq_}, + tstamp{tstamp_} + { cleanup_name();} + + /** + * Get the "display name" for this contact: + * + * If there's a non-empty name, it's Jane Doe <email@example.com> + * otherwise it's just the e-mail address. Names with commas are quoted + * (with the quotes escaped). + * + * @return the display name + */ + std::string display_name() const; + + + /** + * Does the contact contain a valid email address as per + * https://html.spec.whatwg.org/multipage/input.html#valid-e-mail-address + * ? + * + * @return true or false + */ + bool has_valid_email() const; + + /** + * Operator==; based on the hash values (ie. lowercase e-mail address) + * + * @param rhs some other Contact + * + * @return true orf false. + */ + bool operator== (const Contact& rhs) const noexcept { + return hash() == rhs.hash(); + } + + /** + * Get a hash-value for this contact, which gets lazily calculated. This + * * is for use with container classes. This uses the _lowercase_ email + * address. + * + * @return the hash + */ + size_t hash() const { + static size_t cached_hash; + if (cached_hash == 0) { + cached_hash = lowercase_hash(email); + } + return cached_hash; + } + + /** + * Get the corresponding Field::Id (if any) + * for this contact. + * + * @return the field-id or Nothing. + */ + constexpr Option<Field::Id> field_id() const noexcept { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch(type) { + case Type::Bcc: + return Field::Id::Bcc; + case Type::Cc: + return Field::Id::Cc; + case Type::From: + return Field::Id::From; + case Type::To: + return Field::Id::To; + default: + return Nothing; + } +#pragma GCC diagnostic pop + } + + + /* + * data members + */ + + std::string email; /**< Email address for this contact.Not empty */ + std::string name; /**< Name for this contact; can be empty. */ + Type type; /**< Type of contact */ + int64_t message_date; /**< Date of the contact's message */ + bool personal; /**< A personal message? */ + size_t frequency; /**< Frequency of this contact */ + int64_t tstamp; /**< Timestamp for this contact (internal use) */ + +private: + void cleanup_name() { // replace control characters by spaces. + for (auto& c: name) + if (iscntrl(c)) + c = ' '; + } +}; + +constexpr Option<Contact::Type> +contact_type_from_field_id(Field::Id id) noexcept { + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch(id) { + case Field::Id::Bcc: + return Contact::Type::Bcc; + case Field::Id::Cc: + return Contact::Type::Cc; + case Field::Id::From: + return Contact::Type::From; + case Field::Id::To: + return Contact::Type::To; + default: + return Nothing; + } +#pragma GCC diagnostic pop +} + +using Contacts = std::vector<Contact>; + +/** + * Get contacts as a comma-separated list. + * + * @param contacts contacs + * + * @return string with contacts. + */ +std::string to_string(const Contacts& contacts); + +} // namespace Mu + +/** + * Implement our hash int std:: + */ +template<> struct std::hash<Mu::Contact> { + std::size_t operator()(const Mu::Contact& c) const noexcept { + return c.hash(); + } +}; + +#endif /* MU_CONTACT_HH__ */ diff --git a/lib/message/mu-document.cc b/lib/message/mu-document.cc new file mode 100644 index 0000000..428b946 --- /dev/null +++ b/lib/message/mu-document.cc @@ -0,0 +1,497 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "mu-document.hh" +#include "mu-message.hh" +#include "utils/mu-sexp.hh" + +#include <cstdint> +#include <glib.h> +#include <numeric> +#include <algorithm> +#include <charconv> +#include <cinttypes> + +#include <string> +#include <utils/mu-utils.hh> + +using namespace Mu; + +// backward compat +#ifndef HAVE_XAPIAN_FLAG_NGRAMS +#define FLAG_NGRAMS FLAG_CJK_NGRAM +#endif /*HAVE_XAPIAN_FLAG_NGRAMS*/ + + +const Xapian::Document& +Document::xapian_document() const +{ + if (dirty_sexp_) { + xdoc_.set_data(sexp().to_string()); + dirty_sexp_ = false; + } + return xdoc_; +} + +template<typename SexpType> void +Document::put_prop(const std::string& pname, SexpType&& val) +{ + cached_sexp().put_props(pname, std::forward<SexpType>(val)); + dirty_sexp_ = true; +} + +template<typename SexpType> void +Document::put_prop(const Field& field, SexpType&& val) +{ + put_prop(std::string(":") + std::string{field.name}, + std::forward<SexpType>(val)); +} + +static Xapian::TermGenerator +make_term_generator(Xapian::Document& doc, Document::Options opts) +{ + Xapian::TermGenerator termgen; + + if (any_of(opts & Document::Options::SupportNgrams)) + termgen.set_flags(Xapian::TermGenerator::FLAG_NGRAMS); + + termgen.set_document(doc); + + return termgen; +} + +static void +add_search_term(Xapian::Document& doc, const Field& field, const std::string& val, + Document::Options opts) +{ + if (field.is_normal_term() || field.is_phrasable_term()) { + const auto flat{utf8_flatten(val)}; + if (field.is_normal_term()) + doc.add_term(field.xapian_term(flat)); + if (field.is_phrasable_term()) { + auto termgen{make_term_generator(doc, opts)}; + termgen.index_text(flat, 1, field.xapian_term()); + } + } else if (field.is_boolean_term()) { + doc.add_boolean_term(field.xapian_term(val)); + } else + throw std::logic_error("not a search term"); +} + +void +Document::add(Field::Id id, const std::string& val) +{ + const auto field{field_from_id(id)}; + + if (field.is_value()) + xdoc_.add_value(field.value_no(), val); + + if (field.is_searchable()) + add_search_term(xdoc_, field, val, options_); + + if (field.include_in_sexp()) + put_prop(field, val); +} + +void +Document::add(Field::Id id, const std::vector<std::string>& vals) +{ + if (vals.empty()) + return; + + const auto field{field_from_id(id)}; + if (field.is_value()) + xdoc_.add_value(field.value_no(), Mu::join(vals, SepaChar1)); + + if (field.is_searchable()) + std::for_each(vals.begin(), vals.end(), + [&](const auto& val) { + add_search_term(xdoc_, field, val, options_); }); + + if (field.include_in_sexp()) { + Sexp elms{}; + for(auto&& val: vals) + elms.add(val); + put_prop(field, std::move(elms)); + } +} + + +std::vector<std::string> +Document::string_vec_value(Field::Id field_id) const noexcept +{ + return Mu::split(string_value(field_id), SepaChar1); +} + +static Sexp +make_contacts_sexp(const Contacts& contacts) +{ + Sexp contacts_sexp; + + seq_for_each(contacts, [&](auto&& c) { + Sexp contact(":email"_sym, c.email); + if (!c.name.empty()) + contact.add(":name"_sym, c.name); + contacts_sexp.add(std::move(contact)); + }); + + return contacts_sexp; +} + +void +Document::add(Field::Id id, const Contacts& contacts) +{ + if (contacts.empty()) + return; + + const auto field{field_from_id(id)}; + std::vector<std::string> cvec; + + const std::string sepa2(1, SepaChar2); + auto&& termgen{make_term_generator(xdoc_, options_)}; + + for (auto&& contact: contacts) { + + const auto cfield_id{contact.field_id()}; + if (!cfield_id || *cfield_id != id) + continue; + + const auto e{contact.email}; + xdoc_.add_term(field.xapian_term(e)); + + /* allow searching for address components, too */ + const auto atpos = e.find('@'); + if (atpos != std::string::npos && atpos < e.size() - 1) { + xdoc_.add_term(field.xapian_term(e.substr(0, atpos))); + xdoc_.add_term(field.xapian_term(e.substr(atpos + 1))); + } + + if (!contact.name.empty()) + termgen.index_text(utf8_flatten(contact.name), 1, + field.xapian_term()); + cvec.emplace_back(contact.email + sepa2 + contact.name); + } + + if (!cvec.empty()) + xdoc_.add_value(field.value_no(), join(cvec, SepaChar1)); + + if (field.include_in_sexp()) + put_prop(field, make_contacts_sexp(contacts)); +} + +Contacts +Document::contacts_value(Field::Id id) const noexcept +{ + const auto vals{string_vec_value(id)}; + Contacts contacts; + contacts.reserve(vals.size()); + + const auto ctype{contact_type_from_field_id(id)}; + if (G_UNLIKELY(!ctype)) { + mu_critical("invalid field-id for contact-type: <{}>", + static_cast<size_t>(id)); + return {}; + } + + for (auto&& s: vals) { + + const auto pos = s.find(SepaChar2); + if (G_UNLIKELY(pos == std::string::npos)) { + mu_critical("invalid contact data '{}'", s); + break; + } + + contacts.emplace_back(s.substr(0, pos), s.substr(pos + 1), *ctype); + } + + return contacts; +} + +void +Document::add_extra_contacts(const std::string& propname, const Contacts& contacts) +{ + if (!contacts.empty()) { + put_prop(propname, make_contacts_sexp(contacts)); + dirty_sexp_ = true; + } +} + + +static Sexp +make_emacs_time_sexp(::time_t t) +{ + return Sexp().add(static_cast<unsigned>(t >> 16), + static_cast<unsigned>(t & 0xffff), + 0); +} + +void +Document::add(Field::Id id, int64_t val) +{ + /* + * Xapian stores everything (incl. numbers) as strings. + * + * we comply, by storing a number a base-16 and prefixing with 'f' + + * length; such that the strings are sorted in the numerical order. + */ + const auto field{field_from_id(id)}; + + if (field.is_value()) + xdoc_.add_value(field.value_no(), to_lexnum(val)); + + if (field.include_in_sexp()) { + if (field.is_time_t()) + put_prop(field, make_emacs_time_sexp(val)); + else + put_prop(field, val); + } +} + +int64_t +Document::integer_value(Field::Id field_id) const noexcept +{ + if (auto&& v{string_value(field_id)}; v.empty()) + return 0; + else + return from_lexnum(v); +} + +void +Document::add(Priority prio) +{ + constexpr auto field{field_from_id(Field::Id::Priority)}; + + xdoc_.add_value(field.value_no(), std::string(1, to_char(prio))); + xdoc_.add_boolean_term(field.xapian_term(to_char(prio))); + + if (field.include_in_sexp()) + put_prop(field, Sexp::Symbol(priority_name(prio))); +} + +Priority +Document::priority_value() const noexcept +{ + const auto val{string_value(Field::Id::Priority)}; + return priority_from_char(val.empty() ? 'n' : val[0]); +} + +void +Document::add(Flags flags) +{ + constexpr auto field{field_from_id(Field::Id::Flags)}; + + Sexp flaglist; + xdoc_.add_value(field.value_no(), to_lexnum(static_cast<int64_t>(flags))); + flag_infos_for_each([&](auto&& flag_info) { + auto term=[&](){return field.xapian_term(flag_info.shortcut_lower());}; + if (any_of(flag_info.flag & flags)) { + xdoc_.add_boolean_term(term()); + flaglist.add(Sexp::Symbol(flag_info.name)); + } + }); + + if (field.include_in_sexp()) + put_prop(field, std::move(flaglist)); +} + + +Flags +Document::flags_value() const noexcept +{ + return static_cast<Flags>(integer_value(Field::Id::Flags)); +} + +void +Document::remove(Field::Id field_id) +{ + const auto field{field_from_id(field_id)}; + const auto pfx{field.xapian_prefix()}; + + xapian_try([&]{ + + if (auto&& val{xdoc_.get_value(field.value_no())}; !val.empty()) { + // g_debug("removing value<%u>: '%s'", field.value_no(), + // val.c_str()); + xdoc_.remove_value(field.value_no()); + } + + std::vector<std::string> kill_list; + for (auto&& it = xdoc_.termlist_begin(); + it != xdoc_.termlist_end(); ++it) { + const auto term{*it}; + if (!term.empty() && term.at(0) == pfx) + kill_list.emplace_back(term); + } + + for (auto&& term: kill_list) { + // g_debug("removing term '%s'", term.c_str()); + try { + xdoc_.remove_term(term); + } catch(const Xapian::InvalidArgumentError& xe) { + mu_critical("failed to remove '{}'", term); + } + } + }); +} + + +#ifdef BUILD_TESTS + +#include "utils/mu-test-utils.hh" + +#define assert_same_contact(C1,C2) do { \ + g_assert_cmpstr(C1.email.c_str(),==,C2.email.c_str()); \ + g_assert_cmpstr(C2.name.c_str(),==,C2.name.c_str()); \ + } while (0) + +#define assert_same_contacts(CV1,CV2) do { \ + g_assert_cmpuint(CV1.size(),==,CV2.size()); \ + for (auto i = 0U; i != CV1.size(); ++i) \ + assert_same_contact(CV1[i], CV2[i]); \ + } while(0) + + + +static const Contacts test_contacts = {{ + Contact{"john@example.com", "John", Contact::Type::Bcc}, + Contact{"ringo@example.com", "Ringo", Contact::Type::Bcc}, + Contact{"paul@example.com", "Paul", Contact::Type::Cc}, + Contact{"george@example.com", "George", Contact::Type::Cc}, + Contact{"james@example.com", "James", Contact::Type::From}, + Contact{"lars@example.com", "Lars", Contact::Type::To}, + Contact{"kirk@example.com", "Kirk", Contact::Type::To}, + Contact{"jason@example.com", "Jason", Contact::Type::To} + }}; + +static void +test_bcc() +{ + { + Document doc; + doc.add(Field::Id::Bcc, test_contacts); + + Contacts expected_contacts = {{ + Contact{"john@example.com", "John", + Contact::Type::Bcc}, + Contact{"ringo@example.com", "Ringo", + Contact::Type::Bcc}, + }}; + const auto actual_contacts = doc.contacts_value(Field::Id::Bcc); + assert_same_contacts(expected_contacts, actual_contacts); + } + + { + Document doc; + Contacts contacts = {{ + Contact{"john@example.com", "John Lennon", + Contact::Type::Bcc}, + Contact{"ringo@example.com", "Ringo", + Contact::Type::Bcc}, + }}; + doc.add(Field::Id::Bcc, contacts); + + TempDir tempdir; + auto db = Xapian::WritableDatabase(tempdir.path()); + db.add_document(doc.xapian_document()); + + auto contacts2 = doc.contacts_value(Field::Id::Bcc); + assert_same_contacts(contacts, contacts2); + } + +} + +static void +test_cc() +{ + Document doc; + doc.add(Field::Id::Cc, test_contacts); + + Contacts expected_contacts = {{ + Contact{"paul@example.com", "Paul", Contact::Type::Cc}, + Contact{"george@example.com", "George", Contact::Type::Cc} + }}; + const auto actual_contacts = doc.contacts_value(Field::Id::Cc); + + assert_same_contacts(expected_contacts, actual_contacts); +} + + +static void +test_from() +{ + Document doc; + doc.add(Field::Id::From, test_contacts); + + Contacts expected_contacts = {{ + Contact{"james@example.com", "James", Contact::Type::From}, + }}; + const auto actual_contacts = doc.contacts_value(Field::Id::From); + + assert_same_contacts(expected_contacts, actual_contacts); +} + +static void +test_to() +{ + Document doc; + doc.add(Field::Id::To, test_contacts); + + Contacts expected_contacts = {{ + Contact{"lars@example.com", "Lars", Contact::Type::To}, + Contact{"kirk@example.com", "Kirk", Contact::Type::To}, + Contact{"jason@example.com", "Jason", Contact::Type::To} + }}; + const auto actual_contacts = doc.contacts_value(Field::Id::To); + + assert_same_contacts(expected_contacts, actual_contacts); +} + + +static void +test_size() +{ + { + Document doc; + doc.add(Field::Id::Size, 12345); + g_assert_cmpuint(doc.integer_value(Field::Id::Size),==,12345); + } + + { + Document doc; + g_assert_cmpuint(doc.integer_value(Field::Id::Size),==,0); + } +} + + +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/message/document/bcc", test_bcc); + g_test_add_func("/message/document/cc", test_cc); + g_test_add_func("/message/document/from", test_from); + g_test_add_func("/message/document/to", test_to); + + g_test_add_func("/message/document/size", test_size); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-document.hh b/lib/message/mu-document.hh new file mode 100644 index 0000000..5119044 --- /dev/null +++ b/lib/message/mu-document.hh @@ -0,0 +1,261 @@ +/** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_DOCUMENT_HH__ +#define MU_DOCUMENT_HH__ + +#include <utility> +#include <string> +#include <vector> + +#include "mu-xapian-db.hh" +#include "mu-fields.hh" +#include "mu-priority.hh" +#include "mu-flags.hh" +#include "mu-contact.hh" +#include <utils/mu-option.hh> +#include <utils/mu-sexp.hh> + +namespace Mu { + +/** + * A Document describes the information about a message that is + * or can be stored in the database. + * + */ +class Document { +public: + enum struct Options { + None = 0, + SupportNgrams = 1 << 0, /**< Support ngrams, as used in + * CJK and other languages. */ + }; + + /** + * Construct a message for a new Xapian Document + * + * @param flags behavioral flags + */ + Document(Options opts = Options::None): options_{opts} {} + + /** + * Construct a message document based on an existing Xapian document. + * + * @param doc + * @param flags behavioral flags + */ + Document(const Xapian::Document& doc, Options opts = Options::None): + xdoc_{doc}, options_{opts} {} + + /** + * DTOR + */ + ~Document() { + xapian_document(); // for side-effect up updating sexp. + } + + /** + * Get a reference to the underlying Xapian document. + * + */ + const Xapian::Document& xapian_document() const; + + /** + * Get the doc-id for this document + * + * @return the docid + */ + Xapian::docid docid() const { return xdoc_.get_docid(); } + + /* + * updating a document with terms & values + */ + + /** + * Add a string value to the document + * + * @param field_id field id + * @param val string value + */ + void add(Field::Id field_id, const std::string& val); + + /** + * Add a string-vec value to the document, if non-empty + * + * @param field_id field id + * @param val string-vec value + */ + void add(Field::Id field_id, const std::vector<std::string>& vals); + + + /** + * Add message-contacts to the document, if non-empty + * + * @param field_id field id + * @param contacts message contacts + */ + void add(Field::Id id, const Contacts& contacts); + + /** + * Add some extra contacts with the given propname; this is useful for + * ":reply-to" and ":list-post" which don't have a Field::Id and are + * only present in the sexp, not in the terms/values + * + * @param propname property name (e.g.,. ":reply-to") + * @param contacts contacts for this property. + */ + void add_extra_contacts(const std::string& propname, + const Contacts& contacts); + + /** + * Add an integer value to the document + * + * @param field_id field id + * @param val integer value + */ + void add(Field::Id field_id, int64_t val); + + /** + * Add a message priority to the document + * + * @param prio priority + */ + void add(Priority prio); + + + /** + * Add message flags to the document + * + * @param flags mesage flags. + */ + void add(Flags flags); + + /** + * Remove values and terms for some field. + * + * @param field_id + */ + void remove(Field::Id field_id); + + /** + * Get the cached s-expression + * + * @return the cached s-expression + */ + const Sexp& sexp() const { return cached_sexp(); } + + /** + * Get the message s-expression as a string + * + * @return message s-expression string + */ + std::string sexp_str() const { return xdoc_.get_data(); } + + /** + * Generically adds an optional value, if set, to the document + * + * @param id the field 0d + * @param an optional value + */ + template<typename T> void add(Field::Id id, const Option<T>& val) { + if (val) + add(id, val.value()); + } + + /* + * Retrieving values + */ + + /** + * Get a message-field as a string-value + * + * @param field_id id of the field to get. + * + * @return a string (empty if not found) + */ + std::string string_value(Field::Id field_id) const noexcept { + return xapian_try([&]{ + return xdoc_.get_value(field_from_id(field_id).value_no()); + }, std::string{}); + } + + /** + * Get a vec of string values. + * + * @param field_id id of the field to get + * + * @return a string list + */ + std::vector<std::string> string_vec_value(Field::Id field_id) const noexcept; + + + /** + * Get an integer value + * + * @param field_id id of the field to get + * + * @return an integer or 0 if not found. + */ + int64_t integer_value(Field::Id field_id) const noexcept; + + + /** + * Get contacts + * + * @param field_id id of the contacts field to get + * + * @return a contacts list + */ + Contacts contacts_value(Field::Id id) const noexcept; + + /** + * Get the priority + * + * @return the message priority + */ + Priority priority_value() const noexcept; + + /** + * Get the message flags + * + * + * @return flags + */ + Flags flags_value() const noexcept; + +private: + template<typename SexpType> void put_prop(const Field& field, SexpType&& val); + template<typename SexpType> void put_prop(const std::string& pname, SexpType&& val); + + Sexp& cached_sexp() const { + if (cached_sexp_.empty()) + if (auto&& s{Sexp::parse(xdoc_.get_data())}; s) + cached_sexp_ = std::move(*s); + return cached_sexp_; + } + + mutable Xapian::Document xdoc_; + Options options_; + mutable Sexp cached_sexp_; + mutable bool dirty_sexp_{}; /* xdoc's sexp is outdated */ +}; +MU_ENABLE_BITOPS(Document::Options); + +} // namepace Mu + +#endif /* MU_DOCUMENT_HH__ */ diff --git a/lib/message/mu-fields.cc b/lib/message/mu-fields.cc new file mode 100644 index 0000000..f64df5f --- /dev/null +++ b/lib/message/mu-fields.cc @@ -0,0 +1,194 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-fields.hh" +#include "mu-flags.hh" + +#include "utils/mu-test-utils.hh" + +using namespace Mu; + +std::string +Field::xapian_term(const std::string& s) const +{ + const auto start{std::string(1U, xapian_prefix())}; + if (const auto& size = s.size(); size == 0) + return start; + + std::string res{start}; + res.reserve(s.size() + 10); + + /* slightly optimized common pure-ascii. */ + if (G_LIKELY(g_str_is_ascii(s.c_str()))) { + res += s; + for (auto i = 1; res[i]; ++i) + res[i] = g_ascii_tolower(res[i]); + } else + res += utf8_flatten(s); + + if (G_UNLIKELY(res.size() > MaxTermLength)) + res.erase(MaxTermLength); + + return res; +} + +/** + * compile-time checks + */ +constexpr bool +validate_field_ids() +{ + for (auto id = 0U; id != Field::id_size(); ++id) { + const auto field_id = static_cast<Field::Id>(id); + if (field_from_id(field_id).id != field_id) + return false; + } + return true; +} + +constexpr bool +validate_field_shortcuts() +{ +#ifdef BUILD_TESTS + std::array<size_t, 26> no_dups = {0}; +#endif /*BUILD_TESTS*/ + for (auto id = 0U; id != Field::id_size(); ++id) { + const auto field_id = static_cast<Field::Id>(id); + const auto shortcut = field_from_id(field_id).shortcut; + if (shortcut != 0 && + (shortcut < 'a' || shortcut > 'z')) + return false; +#ifdef BUILD_TESTS + if (shortcut != 0) { + if (++no_dups[static_cast<size_t>(shortcut-'a')] > 1) { + mu_critical("shortcut '{}' is duplicated", shortcut); + return false; + } + } +#endif + } + + return true; +} + + +constexpr /*static*/ bool +validate_field_flags() +{ + for (auto&& field: Fields) { + /* - A field has at most one of Phrasable, Boolean */ + size_t flagnum{}; + + if (field.is_phrasable_term()) + ++flagnum; + if (field.is_boolean_term()) + ++flagnum; + + if (flagnum > 1) { + //mu_warning("invalid field {}", field.name); + return false; + } + } + + return true; +} + +/* + * tests... also build as runtime-tests, so we can get coverage info + */ +#ifdef BUILD_TESTS +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + + +[[maybe_unused]] +static void +test_ids() +{ + static_assert(validate_field_ids()); +} + +[[maybe_unused]] +static void +test_shortcuts() +{ + static_assert(validate_field_shortcuts()); +} + +[[maybe_unused]] +static void +test_prefix() +{ + static_assert(field_from_id(Field::Id::Subject).xapian_prefix() == 'S'); +} + +[[maybe_unused]] +static void +test_field_flags() +{ + static_assert(validate_field_flags()); +} + +#ifdef BUILD_TESTS + + +static void +test_field_from_name() +{ + g_assert_true(field_from_name("s")->id == Field::Id::Subject); + g_assert_true(field_from_name("subject")->id == Field::Id::Subject); + g_assert_false(!!field_from_name("8")); + g_assert_false(!!field_from_name("")); + + g_assert_true(field_from_name("").value_or(field_from_id(Field::Id::Bcc)).id == + Field::Id::Bcc); +} + +static void +test_xapian_term() +{ + using namespace std::string_literals; + using namespace std::literals; + + assert_equal(field_from_id(Field::Id::Subject).xapian_term(""s), "S"); + assert_equal(field_from_id(Field::Id::Subject).xapian_term("boo"s), "Sboo"); + + assert_equal(field_from_id(Field::Id::From).xapian_term('x'), "Fx"); + assert_equal(field_from_id(Field::Id::To).xapian_term("boo"sv), "Tboo"); + + auto s1 = field_from_id(Field::Id::Subject).xapian_term(std::string(MaxTermLength - 1, 'x')); + auto s2 = field_from_id(Field::Id::Subject).xapian_term(std::string(MaxTermLength, 'x')); + g_assert_cmpuint(s1.length(), ==, s2.length()); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/message/fields/ids", test_ids); + g_test_add_func("/message/fields/shortcuts", test_shortcuts); + g_test_add_func("/message/fields/from-name", test_field_from_name); + g_test_add_func("/message/fields/prefix", test_prefix); + g_test_add_func("/message/fields/xapian-term", test_xapian_term); + g_test_add_func("/message/fields/flags", test_field_flags); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-fields.hh b/lib/message/mu-fields.hh new file mode 100644 index 0000000..19a222b --- /dev/null +++ b/lib/message/mu-fields.hh @@ -0,0 +1,605 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_FIELDS_HH__ +#define MU_FIELDS_HH__ + +#include <cstdint> +#include <string_view> +#include <algorithm> +#include <array> +#include <mu-xapian-db.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> + +namespace Mu { + +// Xapian does not like terms much longer than this +constexpr auto MaxTermLength = 240; +// http://article.gmane.org/gmane.comp.search.xapian.general/3656 */ + +struct Field { + /** + * Field Ids. + * + * Note, the Ids are also used as indices in the Fields array, + * so their numerical values must be 0...Count. + * + */ + enum struct Id { + Bcc = 0, /**< Blind Carbon-Copy */ + BodyText, /**< Text body */ + Cc, /**< Carbon-Copy */ + Changed, /**< Last change time (think 'ctime') */ + Date, /**< Message date */ + EmbeddedText, /**< Embedded text in message */ + File, /**< Filename */ + Flags, /**< Message flags */ + From, /**< Message sender */ + Language, /**< Body language */ + Maildir, /**< Maildir path */ + MailingList, /**< Mailing list */ + MessageId, /**< Message Id */ + MimeType, /**< MIME-Type */ + Path, /**< File-system Path */ + Priority, /**< Message priority */ + References, /**< All references (incl. Reply-To:) */ + Size, /**< Message size (in bytes) */ + Subject, /**< Message subject */ + Tags, /**< Message Tags */ + ThreadId, /**< Thread Id */ + To, /**< To: recipient */ + // + _count_ /**< Number of Ids */ + }; + + /** + * Get the number of Id values. + * + * @return the number. + */ + static constexpr size_t id_size() + { + return static_cast<size_t>(Id::_count_); + } + + constexpr Xapian::valueno value_no() const { + return static_cast<Xapian::valueno>(id); + } + + /** + * Field types + * + */ + enum struct Type { + String, /**< String */ + StringList, /**< List of strings */ + ContactList, /**< List of contacts */ + ByteSize, /**< Size in bytes */ + TimeT, /**< A time_t value */ + Integer, /**< An integer */ + }; + + constexpr bool is_string() const { return type == Type::String; } + constexpr bool is_string_list() const { return type == Type::StringList; } + constexpr bool is_byte_size() const { return type == Type::ByteSize; } + constexpr bool is_time_t() const { return type == Type::TimeT; } + constexpr bool is_integer() const { return type == Type::Integer; } + constexpr bool is_numerical() const { return is_byte_size() || is_time_t() || is_integer(); } + + /** + * Field flags + * note: the differences for our purposes between a xapian field and a + * term: - there is only a single value for some item in per document + * (msg), ie. one value containing the list of To: addresses - there + * can be multiple terms, each containing e.g. one of the To: + * addresses - searching uses terms, but to display some field, it + * must be in the value + * + * Rules (build-time enforced): + * - A field has at most one of PhrasableTerm, BooleanTerm, ContactTerm. + */ + + enum struct Flag { + /* + * Different kind of terms; at most one is true, and cannot be combined with + * Contact. Compile-time enforced. + */ + NormalTerm = 1 << 0, + /**< Field is a searchable term */ + BooleanTerm = 1 << 1, + /**< Field is a boolean search-term (i.e. at most one per message); + * wildcards do not work */ + PhrasableTerm = 1 << 2, + /**< Field has phrasable/indexable text as term */ + /* + * Contact flag cannot be combined with any of the term flags. + * This is compile-time enforced. + */ + Contact = 1 << 10, + /**< field contains one or more e-mail-addresses */ + Value = 1 << 11, + /**< Field value is stored (so the literal value can be retrieved) */ + + Range = 1 << 21, + + IncludeInSexp = 1 << 24, + /**< whether to include this field in the cached sexp. */ + + /**< whether this is a range field (e.g., date, size)*/ + Internal = 1 << 26 + }; + + constexpr bool any_of(Flag some_flag) const{ + return (static_cast<int>(some_flag) & static_cast<int>(flags)) != 0; + } + + constexpr bool is_phrasable_term() const { return any_of(Flag::PhrasableTerm); } + constexpr bool is_boolean_term() const { return any_of(Flag::BooleanTerm); } + constexpr bool is_normal_term() const { return any_of(Flag::NormalTerm); } + constexpr bool is_searchable() const { return is_phrasable_term() || + is_boolean_term() || + is_normal_term(); } + constexpr bool is_sortable() const { return is_value(); } + + + constexpr bool is_value() const { return any_of(Flag::Value); } + constexpr bool is_internal() const { return any_of(Flag::Internal); } + + constexpr bool is_contact() const { return any_of(Flag::Contact); } + constexpr bool is_range() const { return any_of(Flag::Range); } + + constexpr bool include_in_sexp() const { return any_of(Flag::IncludeInSexp);} + + /** + * Field members + * + */ + Id id; /**< Id of the message field */ + Type type; /**< Type of the message field */ + std::string_view name; /**< Name of the message field */ + std::string_view alias; /**< Alternative name for the message field */ + std::string_view description; /**< Decription of the message field */ + std::string_view example_query; /**< Example query */ + char shortcut; /**< Shortcut for the message field; a..z */ + Flag flags; /**< Flags */ + + /** + * Convenience / helpers + * + */ + + constexpr char xapian_prefix() const { + /* xapian uses uppercase shortcuts; toupper is not constexpr */ + return shortcut == 0 ? 0 : shortcut - ('a' - 'A'); + } + + /** + * Get the xapian term; truncated to MaxTermLength and + * utf8-flattened. + * + * @param s + * + * @return the xapian term + */ + std::string xapian_term(const std::string& s="") const; + std::string xapian_term(std::string_view sv) const { + return xapian_term(std::string{sv}); + } + std::string xapian_term(char c) const { + return xapian_term(std::string(1, c)); + } +}; + +// equality +static inline constexpr bool operator==(const Field& f1, const Field& f2) { return f1.id == f2.id; } +static inline constexpr bool operator==(const Field& f1, const Field::Id id) { return f1.id == id; } + + +MU_ENABLE_BITOPS(Field::Flag); + +/** + * Sequence of _all_ message fields + */ +static constexpr std::array<Field, Field::id_size()> + Fields = { + { + { + Field::Id::Bcc, + Field::Type::ContactList, + "bcc", {}, + "Blind carbon-copy recipient", + "bcc:foo@example.com", + 'h', + Field::Flag::Contact | + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm, + }, + { + Field::Id::BodyText, + Field::Type::String, + "body", {}, + "Message plain-text body", + "body:capybara", + 'b', + Field::Flag::PhrasableTerm, + }, + { + Field::Id::Cc, + Field::Type::ContactList, + "cc", {}, + "Carbon-copy recipient", + "cc:quinn@example.com", + 'c', + Field::Flag::Contact | + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm, + }, + { + Field::Id::Changed, + Field::Type::TimeT, + "changed", {}, + "Last change time", + "changed:30M..", + 'k', + Field::Flag::Value | + Field::Flag::Range | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Date, + Field::Type::TimeT, + "date", {}, + "Message date", + "date:20220101..20220505", + 'd', + Field::Flag::Value | + Field::Flag::Range | + Field::Flag::IncludeInSexp + }, + { + Field::Id::EmbeddedText, + Field::Type::String, + "embed", {}, + "Embedded text", + "embed:war OR embed:peace", + 'e', + Field::Flag::PhrasableTerm + }, + { + Field::Id::File, + Field::Type::String, + "file", {}, + "Attachment file name", + "file:/image\\.*.jpg/", + 'j', + Field::Flag::BooleanTerm + }, + { + Field::Id::Flags, + Field::Type::Integer, + "flags", "flag", + "Message properties", + "flag:unread AND flag:personal", + 'g', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::From, + Field::Type::ContactList, + "from", {}, + "Message sender", + "from:jimbo", + 'f', + Field::Flag::Contact | + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm, + }, + { + Field::Id::Language, + Field::Type::String, + "language", "lang", + "ISO 639-1 language code for body", + "lang:nl", + 'a', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Maildir, + Field::Type::String, + "maildir", {}, + "Maildir path for message", + "maildir:/private/archive", + 'm', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::MailingList, + Field::Type::String, + "list", {}, + "Mailing list (List-Id:)", + "list:mu-discuss.example.com", + 'v', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::MessageId, + Field::Type::String, + "message-id", "msgid", + "Message-Id", + "msgid:abc@123", + 'i', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::MimeType, + Field::Type::String, + "mime", "mime-type", + "Attachment MIME-type", + "mime:image/jpeg", + 'y', + Field::Flag::BooleanTerm + }, + { + Field::Id::Path, + Field::Type::String, + "path", {}, + "File system path to message", + "path:/a/b/Maildir/cur/msg:2,S", + 'l', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Priority, + Field::Type::Integer, + "priority", "prio", + "Priority", + "prio:high", + 'p', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::References, + Field::Type::StringList, + "references", {}, + "References to related messages", + {}, + 'r', + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Size, + Field::Type::ByteSize, + "size", {}, + "Message size in bytes", + "size:1M..5M", + 'z', + Field::Flag::Value | + Field::Flag::Range | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Subject, + Field::Type::String, + "subject", {}, + "Message subject", + "subject:wombat", + 's', + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm + }, + { + Field::Id::Tags, + Field::Type::StringList, + "tags", "tag", + "Message tags", + "tag:projectx", + 'x', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::ThreadId, + Field::Type::String, + "thread", {}, + "Thread a message belongs to", + {}, + 'w', + Field::Flag::BooleanTerm | + Field::Flag::Value + }, + { + Field::Id::To, + Field::Type::ContactList, + "to", {}, + "Message recipient", + "to:flimflam@example.com", + 't', + Field::Flag::Contact | + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm, + }, + }}; + +/* + * Convenience + */ + +/** + * Get the message field for the given Id. + * + * @param id of the message field + * + * @return ref of the message field. + */ +constexpr const Field& +field_from_id(Field::Id id) +{ + return Fields.at(static_cast<size_t>(id)); +} + +/** + * Invoke func for each message-field + * + * @param func some callable + */ +template <typename Func> +constexpr void field_for_each(Func&& func) { + for (const auto& field: Fields) + func(field); +} + +/** + * Find a message field that satisfies some predicate + * + * @param pred the predicate (a callable) + * + * @return a message-field id, or nullopt if not found. + */ +template <typename Pred> +constexpr Option<Field> field_find_if(Pred&& pred) { + for (auto&& field: Fields) + if (pred(field)) + return field; + return Nothing; +} + +/** + * Get the the message-field for the given name or shortcut + * + * @param name_or_shortcut + * + * @return the message-field or Nothing + */ +static inline +Option<Field> field_from_shortcut(char shortcut) { + return field_find_if([&](auto&& field){ + return field.shortcut == shortcut; + }); +} +static inline +Option<Field> field_from_name(const std::string& name) { + switch(name.length()) { + case 0: + return Nothing; + case 1: + return field_from_shortcut(name[0]); + default: + return field_find_if([&](auto&& field){ + return name == field.name || name == field.alias; + }); + } +} + +/** + * Return combination-fields such + * as "contact", "recip" and "" (empty) + * + * @param name combination field name + * + * @return list of matching fields + */ +using FieldsVec = std::vector<Field>; +static inline +const FieldsVec& fields_from_name(const std::string& name) { + + static const FieldsVec none{}; + static const FieldsVec recip_fields ={ + field_from_id(Field::Id::To), + field_from_id(Field::Id::Cc), + field_from_id(Field::Id::Bcc)}; + + static const FieldsVec contact_fields = { + field_from_id(Field::Id::To), + field_from_id(Field::Id::Cc), + field_from_id(Field::Id::Bcc), + field_from_id(Field::Id::From), + }; + static const FieldsVec empty_fields= { + field_from_id(Field::Id::To), + field_from_id(Field::Id::Cc), + field_from_id(Field::Id::Bcc), + field_from_id(Field::Id::From), + field_from_id(Field::Id::Subject), + field_from_id(Field::Id::BodyText), + field_from_id(Field::Id::EmbeddedText), + }; + + if (name == "recip") + return recip_fields; + else if (name == "contact") + return contact_fields; + else if (name.empty()) + return empty_fields; + else + return none; +} + +static inline bool +field_is_combi (const std::string& name) +{ + return name == "recip" || name == "contact"; +} + + +/** + * Get the Field::Id for some number, or nullopt if it does not match + * + * @param id an id number + * + * @return Field::Id or nullopt + */ +static inline +Option<Field> field_from_number(size_t id) +{ + if (id >= static_cast<size_t>(Field::Id::_count_)) + return Nothing; + else + return field_from_id(static_cast<Field::Id>(id)); +} + + +} // namespace Mu +#endif /* MU_FIELDS_HH__ */ diff --git a/lib/message/mu-flags.cc b/lib/message/mu-flags.cc new file mode 100644 index 0000000..7ff340e --- /dev/null +++ b/lib/message/mu-flags.cc @@ -0,0 +1,196 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +/* + * implementation is almost completely in the header; here we just add some + * compile-time tests. + */ + +#include "mu-flags.hh" + +using namespace Mu; + +std::string +Mu::to_string(Flags flags) +{ + std::string str; + + for (auto&& info: AllMessageFlagInfos) + if (any_of(info.flag & flags)) + str+=info.shortcut; + + return str; +} + + +/* + * flags & flag-info + */ +constexpr bool +validate_message_info_flags() +{ + for (auto id = 0U; id != AllMessageFlagInfos.size(); ++id) { + const auto flag = static_cast<Flags>(1 << id); + if (flag != AllMessageFlagInfos[id].flag) + return false; + } + return true; +} + + +/* + * tests... also build as runtime-tests, so we can get coverage info + */ +#ifdef BUILD_TESTS +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + +[[maybe_unused]] static void +test_basic() +{ + static_assert(AllMessageFlagInfos.size() == + __builtin_ctz(static_cast<unsigned>(Flags::_final_))); + static_assert(validate_message_info_flags()); + + static_assert(!!flag_info(Flags::Encrypted)); + static_assert(!flag_info(Flags::None)); + static_assert(!flag_info(static_cast<Flags>(0))); + static_assert(!flag_info(static_cast<Flags>(1<<AllMessageFlagInfos.size()))); +} + +/* + * flag_info + */ +[[maybe_unused]] static void +test_flag_info() +{ + static_assert(flag_info('D')->flag == Flags::Draft); + static_assert(flag_info('l')->flag == Flags::MailingList); + static_assert(!flag_info('y')); + + static_assert(flag_info("trashed")->flag == Flags::Trashed); + static_assert(flag_info("attach")->flag == Flags::HasAttachment); + static_assert(!flag_info("fnorb")); + + + static_assert(flag_info('D')->shortcut_lower() == 'd'); + static_assert(flag_info('u')->shortcut_lower() == 'u'); +} + +/* + * flags_from_expr + */ +[[maybe_unused]] static void +test_flags_from_expr() +{ + static_assert(flags_from_absolute_expr("SRP").value() == + (Flags::Seen | Flags::Replied | Flags::Passed)); + static_assert(flags_from_absolute_expr("Faul").value() == + (Flags::Flagged | Flags::Unread | + Flags::HasAttachment | Flags::MailingList)); + + /* note: unread is a special flag, _implied_ from "new or not seen" */ + static_assert(flags_from_absolute_expr("N").value() == (Flags::New|Flags::Unread)); + + static_assert(!flags_from_absolute_expr("DRT?")); + static_assert(flags_from_absolute_expr("DRT?", true/*ignore invalid*/).value() == + (Flags::Draft | Flags::Replied | + Flags::Trashed | Flags::Unread)); + static_assert(flags_from_absolute_expr("DFPNxulabcdef", true/*ignore invalid*/).value() == + (Flags::Draft|Flags::Flagged|Flags::Passed| + Flags::New | Flags::Encrypted | + Flags::Unread | Flags::MailingList | Flags::Calendar | + Flags::HasAttachment)); +} + + +/* + * flags_from_delta_expr + */ +[[maybe_unused]] static void +test_flags_from_delta_expr() +{ + static_assert(flags_from_delta_expr( + "+S-u-N", Flags::New|Flags::Unread).value() == + Flags::Seen); + + /* note: unread is a special flag, _implied_ from "new or not seen" */ + static_assert(flags_from_delta_expr( + "+S-N", Flags::New|Flags::Unread).value() == + Flags::Seen); + static_assert(flags_from_delta_expr( + "-S", Flags::Seen).value() == + Flags::Unread); + + static_assert(flags_from_delta_expr("+R+P-F", Flags::Seen).value() == + (Flags::Seen|Flags::Passed|Flags::Replied)); + /* '-B' is invalid */ + static_assert(!flags_from_delta_expr("+R+P-B", Flags::Seen)); + /* '-B' is invalid, but ignore invalid */ + static_assert(flags_from_delta_expr("+R+P-B", Flags::Seen, true) == + (Flags::Replied|Flags::Passed|Flags::Seen)); + static_assert(flags_from_delta_expr("+F+T-S", Flags::None, true).value() == + (Flags::Flagged|Flags::Trashed|Flags::Unread)); +} + +/* + * flags_filter + */ +[[maybe_unused]] static void +test_flags_filter() +{ + static_assert(flags_filter(flags_from_absolute_expr( + "DFPNxulabcdef", true/*ignore invalid*/).value(), + MessageFlagCategory::Mailfile) == + (Flags::Draft|Flags::Flagged|Flags::Passed)); +} + + + +[[maybe_unused]] static void +test_flags_keep_unmutable() +{ + static_assert(flags_keep_unmutable((Flags::Seen|Flags::Passed), + (Flags::Flagged|Flags::Draft), + Flags::Replied) == + (Flags::Flagged|Flags::Draft)); +} + + + +#ifdef BUILD_TESTS +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/message/flags/basic", test_basic); + g_test_add_func("/message/flags/flag-info", test_flag_info); + g_test_add_func("/message/flags/flags-from-absolute-expr", + test_flags_from_expr); + g_test_add_func("/message/flags/flags-from-delta-expr", + test_flags_from_delta_expr); + g_test_add_func("/message/flags/flags-filter", + test_flags_filter); + g_test_add_func("/message/flags/flags-keep-unmutable", + test_flags_keep_unmutable); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-flags.hh b/lib/message/mu-flags.hh new file mode 100644 index 0000000..8e424dd --- /dev/null +++ b/lib/message/mu-flags.hh @@ -0,0 +1,422 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_FLAGS_HH__ +#define MU_FLAGS_HH__ + +#include <algorithm> +#include <string_view> +#include <array> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> + +namespace Mu { + +enum struct Flags { + None = 0, /**< No flags */ + /** + * next 6 are seen in the file-info part of maildir message file + * names, ie., in a name like "1234345346:2,<fileinfo>", + * <fileinfo> consists of zero or more of the following + * characters (in ascii order) + */ + Draft = 1 << 0, /**< A draft message */ + Flagged = 1 << 1, /**< A flagged message */ + Passed = 1 << 2, /**< A passed (forwarded) message */ + Replied = 1 << 3, /**< A replied message */ + Seen = 1 << 4, /**< A seen (read) message */ + Trashed = 1 << 5, /**< A trashed message */ + + /** + * decides on cur/ or new/ in the maildir + */ + New = 1 << 6, /**< A new message */ + + /** + * content flags -- not visible in the filename, but used for + * searching + */ + Signed = 1 << 7, /**< Cryptographically signed */ + Encrypted = 1 << 8, /**< Encrypted */ + HasAttachment = 1 << 9, /**< Has an attachment */ + + Unread = 1 << 10, /**< Unread; pseudo-flag, only for queries, so we can + * search for flag:unread, which is equivalent to + * 'flag:new OR NOT flag:seen' */ + /** + * other content flags + */ + MailingList = 1 << 11, /**< A mailing-list message */ + Personal = 1 << 12, /**< A personal message (i.e., at least one of the + * contact fields contains a personal address) */ + Calendar = 1 << 13, /**< A calendar invitation */ + /* + * <private> + */ + _final_ = 1 << 14 +}; +MU_ENABLE_BITOPS(Flags); + +/** + * Message flags category + * + */ +enum struct MessageFlagCategory { + None, /**< Nothing */ + Mailfile, /**< Flag for a message file */ + Maildir, /**< Flag for message file's location */ + Content, /**< Message content flag */ + Pseudo /**< Pseudo flag */ +}; + +/** + * Info about invidual message flags + * + */ +struct MessageFlagInfo { + + Flags flag; /**< The message flag */ + char shortcut; /**< Shortcut character; + * tolower(shortcut) must be + * unique for all flags */ + std::string_view name; /**< Name of the flag */ + MessageFlagCategory category; /**< Flag category */ + std::string_view description; /**< Description */ + + /** + * Get the lower-case version of shortcut + * + * @return lower-case shortcut + */ + constexpr char shortcut_lower() const { + return shortcut >= 'A' && shortcut <= 'Z' ? + shortcut + ('a' - 'A') : shortcut; + } +}; + +/** + * Array of all flag information. + */ +constexpr std::array<MessageFlagInfo, 14> AllMessageFlagInfos = {{ + MessageFlagInfo{Flags::Draft, 'D', "draft", MessageFlagCategory::Mailfile, + "Draft (in progress)" + }, + MessageFlagInfo{Flags::Flagged, 'F', "flagged", MessageFlagCategory::Mailfile, + "User-flagged" + }, + MessageFlagInfo{Flags::Passed, 'P', "passed", MessageFlagCategory::Mailfile, + "Forwarded message" + }, + MessageFlagInfo{Flags::Replied, 'R', "replied", MessageFlagCategory::Mailfile, + "Replied-to" + }, + MessageFlagInfo{Flags::Seen, 'S', "seen", MessageFlagCategory::Mailfile, + "Viewed at least once" + }, + MessageFlagInfo{Flags::Trashed, 'T', "trashed", MessageFlagCategory::Mailfile, + "Marked for deletion" + }, + MessageFlagInfo{Flags::New, 'N', "new", MessageFlagCategory::Maildir, + "New message" + }, + MessageFlagInfo{Flags::Signed, 'z', "signed", MessageFlagCategory::Content, + "Cryptographically signed" + }, + MessageFlagInfo{Flags::Encrypted, 'x', "encrypted", MessageFlagCategory::Content, + "Encrypted" + }, + MessageFlagInfo{Flags::HasAttachment,'a', "attach", MessageFlagCategory::Content, + "Has at least one attachment" + }, + MessageFlagInfo{Flags::Unread, 'u', "unread", MessageFlagCategory::Pseudo, + "New or not seen message" + }, + MessageFlagInfo{Flags::MailingList, 'l', "list", MessageFlagCategory::Content, + "Mailing list message" + }, + MessageFlagInfo{Flags::Personal, 'q', "personal", MessageFlagCategory::Content, + "Personal message" + }, + MessageFlagInfo{Flags::Calendar, 'c', "calendar", MessageFlagCategory::Content, + "Calendar invitation" + }, +}}; + + +/** + * Invoke some callable Func for each flag info + * + * @param func some callable + */ +template<typename Func> +constexpr void flag_infos_for_each(Func&& func) +{ + for (auto&& info: AllMessageFlagInfos) + func(info); +} + +/** + * Get flag info for some flag + * + * @param flag a singular flag + * + * @return the MessageFlagInfo, or Nothing in case of error. + */ +constexpr const Option<MessageFlagInfo> +flag_info(Flags flag) +{ + constexpr auto upper = static_cast<unsigned>(Flags::_final_); + const auto val = static_cast<unsigned>(flag); + + if (__builtin_popcount(val) != 1 || val >= upper) + return Nothing; + + return AllMessageFlagInfos[static_cast<unsigned>(__builtin_ctz(val))]; +} + +/** + * Get flag info for some flag + * + * @param shortcut shortcut character + * + * @return the MessageFlagInfo + */ +constexpr const Option<MessageFlagInfo> +flag_info(char shortcut) +{ + for (auto&& info : AllMessageFlagInfos) + if (info.shortcut == shortcut) + return info; + + return Nothing; +} + +/** + * Get flag info for some flag, either by its name of is shortcut + * + * @param name the name of the message-flag, or its shortcut + * + * @return the MessageFlagInfo or Nothing if not found + */ +constexpr const Option<MessageFlagInfo> +flag_info(std::string_view name) +{ + if (name.empty()) + return Nothing; + + for (auto&& info : AllMessageFlagInfos) + if (info.name == name) + return info; + + return flag_info(name.at(0)); +} + +/** + * 'unread' is a pseudo-flag that means 'new or not seen' + * + * @param flags + * + * @return flags with unread added or removed. + */ +constexpr Flags +imply_unread(Flags flags) +{ + /* unread is a pseudo flag equivalent to 'new or not seen' */ + if (any_of(flags & Flags::New) || none_of(flags & Flags::Seen)) + return flags | Flags::Unread; + else + return flags & ~Flags::Unread; +} + +/** + * There are two string-based expression types for flags: + * 1) 'absolute': replace the existing flags + * 2) 'delta' : flags as a delta of existing flags. + */ + +/** + * Get the (OR'ed) flags corresponding to an expression. + * + * @param expr the expression (a sequence of flag shortcut characters) + * @param ignore_invalid if @true, ignore invalid flags, otherwise return + * nullopt if an invalid flag is encountered + * + * @return the (OR'ed) flags or Flags::None + */ +constexpr Option<Flags> +flags_from_absolute_expr(std::string_view expr, bool ignore_invalid = false) +{ + Flags flags{Flags::None}; + + for (auto&& kar : expr) { + if (const auto& info{flag_info(kar)}; !info) { + if (!ignore_invalid) + return Nothing; + } else + flags |= info->flag; + } + + return imply_unread(flags); +} + +/** + * Calculate flags from existing flags and a delta expression + * + * Update @p flags with the flags in @p expr, where @p exprt consists of the the + * normal flag shortcut characters, prefixed with either '+' or '-', which means + * resp. "add this flag" or "remove this flag". + * + * So, e.g. "-N+S" would unset the NEW flag and set the SEEN flag, without + * affecting other flags. + * + * @param expr delta expression + * @param flags existing flags + * @param ignore_invalid if @true, ignore invalid flags, otherwise return + * Nothing if an invalid flag is encountered + * + * @return new flags, or Nothing in case of error + */ +constexpr Option<Flags> +flags_from_delta_expr(std::string_view expr, Flags flags, + bool ignore_invalid = false) +{ + if (expr.size() % 2 != 0) + return Nothing; + + for (auto u = 0U; u != expr.size(); u += 2) { + if (const auto& info{flag_info(expr[u + 1])}; !info) { + if (!ignore_invalid) + return Nothing; + } else { + switch (expr[u]) { + case '+': flags |= info->flag; break; + case '-': flags &= ~info->flag; break; + default: + if (!ignore_invalid) + return Nothing; + break; + } + } + } + + return imply_unread(flags); +} + +/** + * Calculate the flags from either 'absolute' or 'delta' expressions + * + * @param expr a flag expression, either 'delta' or 'absolute' + * @param flags optional: existing flags or none. Required for delta. + * + * @return either messages flags or Nothing in case of error. + */ +constexpr Option<Flags> +flags_from_expr(std::string_view expr, Option<Flags> flags = Nothing) +{ + if (expr.empty()) + return Nothing; + + if (expr[0] == '+' || expr[0] == '-') + return flags_from_delta_expr( + expr, flags.value_or(Flags::None), true); + else + return flags_from_absolute_expr(expr, true); +} + +/** + * Filter out flags which are not in the given category + * + * @param flags flags + * @param cat category + * + * @return filtered flags + */ +constexpr Flags +flags_filter(Flags flags, MessageFlagCategory cat) +{ + for (auto&& info : AllMessageFlagInfos) + if (info.category != cat) + flags &= ~info.flag; + return flags; +} + +/** + * Filter out any flags which are _not_ Maildir / Mailfile flags + * + * @param flags flags + * + * @return filtered flags + */ +constexpr Flags +flags_maildir_file(Flags flags) +{ + for (auto&& info : AllMessageFlagInfos) + if (info.category != MessageFlagCategory::Maildir && + info.category != MessageFlagCategory::Mailfile) + flags &= ~info.flag; + return flags; +} + + + + +/** + * Return flags, where flags = new_flags but with unmutable_flag in the + * result the same as in old_flags + * + * @param old_flags + * @param new_flags + * @param immutable_flag + * + * @return + */ +constexpr Flags +flags_keep_unmutable(Flags old_flags, Flags new_flags, Flags immutable_flag) +{ + if (any_of(old_flags & immutable_flag)) + return new_flags | immutable_flag; + else + return new_flags & ~immutable_flag; +} + + +/** + * Get a string representation of flags + * + * @param flags flags + * + * @return string as a sequence of message-flag shortcuts + */ +std::string to_string(Flags flags); + + +/** + * Get a string representation of Flags for fmt + * + * @param flags flags + * + * @return string as a sequence of message-flag shortcuts + */ +static inline auto format_as(const Flags& flags) { + return to_string(flags); +} + +} // namespace Mu + +#endif /* MU_FLAGS_HH__ */ diff --git a/lib/message/mu-message-file.cc b/lib/message/mu-message-file.cc new file mode 100644 index 0000000..b077c3b --- /dev/null +++ b/lib/message/mu-message-file.cc @@ -0,0 +1,198 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb.bulk@gmail.com> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-message-file.hh" +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +Result<std::string> +Mu::maildir_from_path(const std::string& path, const std::string& root) +{ + const auto pos = path.find(root); + if (pos != 0 || path[root.length()] != '/') + return Err(Error{Error::Code::InvalidArgument, + "root '{}' is not a root for path '{}'", root, path}); + + auto mdir{path.substr(root.length())}; + auto slash{mdir.rfind('/')}; + + if (G_UNLIKELY(slash == std::string::npos) || slash < 4) + return Err(Error{Error::Code::InvalidArgument, + "invalid path: {}", path}); + mdir.erase(slash); + auto subdir = mdir.data() + slash - 4; + if (G_UNLIKELY(strncmp(subdir, "/cur", 4) != 0 && strncmp(subdir, "/new", 4))) + return Err(Error::Code::InvalidArgument, + "cannot find '/new' or '/cur' - invalid path: {}", path); + + if (mdir.length() == 4) + return "/"; + + mdir.erase(mdir.length() - 4); + + return Ok(std::move(mdir)); +} + +Mu::FileParts +Mu::message_file_parts(const std::string& file) +{ + const auto pos{file.find_last_of(":!;")}; + + /* no suffix at all? */ + if (pos == std::string::npos || + pos > file.length() - 3 || + file[pos + 1] != '2' || + file[pos + 2] != ',') + return FileParts{ file, ':', {}}; + + return FileParts { + file.substr(0, pos), + file[pos], + file.substr(pos + 3) + }; +} + +Mu::Result<DirFile> +Mu::base_message_dir_file(const std::string& path) +{ + constexpr auto newdir{"/new"}; + + const auto dname{dirname(path)}; + bool is_new{!!g_str_has_suffix(dname.c_str(), newdir)}; + + std::string mdir{dname.substr(0, dname.size() - 4)}; + return Ok(DirFile{std::move(mdir), basename(path), is_new}); +} + +Mu::Result<Mu::Flags> +Mu::flags_from_path(const std::string& path) +{ /* + * this gets us the source maildir filesystem path, the directory + * in which new/ & cur/ lives, and the source file + */ + auto dirfile{base_message_dir_file(path)}; + if (!dirfile) + return Err(std::move(dirfile.error())); + + /* a message under new/ is just.. New. Filename is not considered */ + if (dirfile->is_new) + return Ok(Flags::New); + + /* it's cur/ message, so parse the file name */ + const auto parts{message_file_parts(dirfile->file)}; + auto flags{flags_from_absolute_expr(parts.flags_suffix, + true/*ignore invalid*/)}; + if (!flags) { + /* LCOV_EXCL_START*/ + return Err(Error{Error::Code::InvalidArgument, + "invalid flags ('{}')", parts.flags_suffix}); + /* LCOV_EXCL_STOP*/ + } + + /* of course, only _file_ flags are allowed */ + return Ok(flags_filter(flags.value(), MessageFlagCategory::Mailfile)); +} + + +#ifdef BUILD_TESTS + +#include "utils/mu-test-utils.hh" + +static void +test_maildir_from_path() +{ + std::array<std::tuple<std::string, std::string, std::string>, 1> test_cases = {{ + { "/home/foo/Maildir/hello/cur/msg123", "/home/foo/Maildir", "/hello" } + }}; + + for(auto&& tcase: test_cases) { + const auto res{maildir_from_path(std::get<0>(tcase), std::get<1>(tcase))}; + assert_valid_result(res); + assert_equal(*res, std::get<2>(tcase)); + } + + g_assert_false(!!maildir_from_path("/home/foo/Maildir/cur/test1", "/home/bar")); + g_assert_false(!!maildir_from_path("/x", "/x/y")); + g_assert_false(!!maildir_from_path("/home/a/Maildir/b/xxx/test", "/home/a/Maildir")); +} + +static void +test_base_message_dir_file() +{ + struct TestCase { + const std::string path; + DirFile expected; + }; + std::array<TestCase, 1> test_cases = {{ + { "/home/djcb/Maildir/foo/cur/msg:2,S", + { "/home/djcb/Maildir/foo", "msg:2,S", false } } + }}; + for(auto&& tcase: test_cases) { + const auto res{base_message_dir_file(tcase.path)}; + assert_valid_result(res); + assert_equal(res->dir, tcase.expected.dir); + assert_equal(res->file, tcase.expected.file); + g_assert_cmpuint(res->is_new, ==, tcase.expected.is_new); + } +} + +static void +test_flags_from_path() +{ + std::array<std::pair<std::string, Flags>, 5> test_cases = {{ + {"/home/foo/Maildir/test/cur/123456:2,FSR", + (Flags::Replied | Flags::Seen | Flags::Flagged)}, + {"/home/foo/Maildir/test/new/123456", Flags::New}, + {/* NOTE: when in new/, the :2,.. stuff is ignored */ + "/home/foo/Maildir/test/new/123456:2,FR", + Flags::New}, + {"/home/foo/Maildir/test/cur/123456:2,DTP", + (Flags::Draft | Flags::Trashed | Flags::Passed)}, + {"/home/foo/Maildir/test/cur/123456:2,S", Flags::Seen} + }}; + + for (auto&& tcase: test_cases) { + auto res{flags_from_path(tcase.first)}; + assert_valid_result(res); + /* LCOV_EXCL_START*/ + if (g_test_verbose()) { + mu_println("{} -> <{}>", tcase.first, + to_string(res.value())); + g_assert_true(res.value() == tcase.second); + } + /*LCOV_EXCL_STOP*/ + } +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/message/file/maildir-from-path", + test_maildir_from_path); + g_test_add_func("/message/file/base-message-dir-file", + test_base_message_dir_file); + g_test_add_func("/message/file/flags-from-path", test_flags_from_path); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-message-file.hh b/lib/message/mu-message-file.hh new file mode 100644 index 0000000..09a9ed3 --- /dev/null +++ b/lib/message/mu-message-file.hh @@ -0,0 +1,98 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb.bulk@gmail.com> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MESSAGE_FILE_HH__ +#define MU_MESSAGE_FILE_HH__ + +#include "mu-flags.hh" +#include <utils/mu-result.hh> + +namespace Mu { + +/* + * The file-components, ie. + * 1631819685.fb7b279bbb0a7b66.evergrey:2,RS + * => { + * "1631819685.fb7b279bbb0a7b66.evergrey", + * ':', + * "2,", + * "RS" + * } + */ +struct FileParts { + std::string base; /**< basename */ + char separator; /**< separator */ + std::string flags_suffix; /**< suffix (with flags) */ +}; + +/** + * Get the file-parts for some message-file + * + * @param file path to some message file (does not have to exist) + * + * @return FileParts for the message file + */ +FileParts message_file_parts(const std::string& file); + + +struct DirFile { + std::string dir; + std::string file; + bool is_new; +}; + +/** + * Get information about the message file componemts + * + * @param path message path + * + * @return the components for the message file or an error. + */ +Result<DirFile> base_message_dir_file(const std::string& path); + + + +/** + * Get the Maildir flags from the full path of a mailfile. The flags are as + * specified in http://cr.yp.to/proto/maildir.html, plus Flag::New for new + * messages, ie the ones that live in new/. The flags are logically OR'ed. Note + * that the file does not have to exist; the flags are based on the path only. + * + * @param pathname of a mailfile; it does not have to refer to an + * actual message + * + * @return the message flags or an error + */ +Result<Flags> flags_from_path(const std::string& pathname); + +/** + * get the maildir for a certain message path, ie, the path *before* + * cur/ or new/ and *after* the root. + * + * @param path path for some message + * @param root filesystem root for the maildir + * + * @return the maildir or an Error + */ +Result<std::string> maildir_from_path(const std::string& path, + const std::string& root); +} // Mu + + +#endif /* MU_MESSAGE_FILE_HH__ */ diff --git a/lib/message/mu-message-part.cc b/lib/message/mu-message-part.cc new file mode 100644 index 0000000..d1c7ac5 --- /dev/null +++ b/lib/message/mu-message-part.cc @@ -0,0 +1,256 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#include "mu-message-part.hh" +#include "mu-mime-object.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include <string> + +using namespace Mu; + +MessagePart::MessagePart(const Mu::MimeObject& obj): + mime_obj{std::make_unique<Mu::MimeObject>(obj)} +{} + +MessagePart::MessagePart(const MessagePart& other): + MessagePart(*other.mime_obj) +{} + +MessagePart::~MessagePart() = default; + +const MimeObject& +MessagePart::mime_object() const noexcept +{ + return *mime_obj; +} + +static std::string +cook(const std::string& fname, const std::vector<char>& forbidden) +{ + std::string clean; + clean.reserve(fname.length()); + + for (auto& c: basename(fname)) + if (seq_some(forbidden,[&](char fc){return ::iscntrl(c) || c == fc;})) + clean += '-'; + else + clean += c; + + if (clean[0] == '.' && (clean == "." || clean == "..")) + return "-"; + else + return clean; +} + +static std::string +cook_minimal(const std::string& fname) +{ + return cook(fname, { '/' }); +} + +static std::string +cook_full(const std::string& fname) +{ + auto cooked = cook(fname, { '/', ' ', '\\', ':' }); + if (cooked.size() > 1 && cooked[0] == '-') + cooked.erase(0, 1); + + return cooked; +} + +Option<std::string> +MessagePart::cooked_filename(bool minimal) const noexcept +{ + auto&& cooker{minimal ? cook_minimal : cook_full}; + + // a MimePart... use the name if there is one. + if (mime_object().is_part()) + return MimePart{mime_object()}.filename().map(cooker); + + // MimeMessagepart. Construct a name based on subject. + if (mime_object().is_message_part()) { + auto msg{MimeMessagePart{mime_object()}.get_message()}; + if (!msg) + return Nothing; + else + return msg->subject() + .map(cooker) + .value_or("no-subject") + ".eml"; + } + + return Nothing; +} + +Option<std::string> +MessagePart::raw_filename() const noexcept +{ + if (!mime_object().is_part()) + return Nothing; + else + return MimePart{mime_object()}.filename(); +} + + + +Option<std::string> +MessagePart::mime_type() const noexcept +{ + if (const auto ctype{mime_object().content_type()}; ctype) + return ctype->media_type() + "/" + ctype->media_subtype(); + else + return Nothing; +} + +Option<std::string> +MessagePart::content_description() const noexcept +{ + if (!mime_object().is_part()) + return Nothing; + else + return MimePart{mime_object()}.content_description(); +} + +size_t +MessagePart::size() const noexcept +{ + if (!mime_object().is_part()) + return 0; + else + return MimePart{mime_object()}.size(); +} + +bool +MessagePart::is_attachment() const noexcept +{ + if (!mime_object().is_part()) + return false; + else + return MimePart{mime_object()}.is_attachment(); +} + + +Option<std::string> +MessagePart::to_string() const noexcept +{ + if (mime_object().is_part()) + return MimePart{mime_object()}.to_string(); + else + return mime_object().to_string_opt(); +} + +Result<size_t> +MessagePart::to_file(const std::string& path, bool overwrite) const noexcept +{ + if (mime_object().is_part()) + return MimePart{mime_object()}.to_file(path, overwrite); + else if (mime_object().is_message_part()) { + if (auto&& msg{MimeMessagePart{mime_object()}.get_message()}; !msg) + return Err(Error::Code::Message, "failed to get message-part"); + else + return msg->to_file(path, overwrite); + } else + return mime_object().to_file(path, overwrite); +} + +bool +MessagePart::is_signed() const noexcept +{ + return mime_object().is_multipart_signed(); +} + +bool +MessagePart::is_encrypted() const noexcept +{ + return mime_object().is_multipart_encrypted(); +} + +bool /* heuristic */ +MessagePart::looks_like_attachment() const noexcept +{ + auto matches=[](const MimeContentType& ctype, + const std::initializer_list<std::pair<const char*, const char*>>& ctypes) { + return std::find_if(ctypes.begin(), ctypes.end(), [&](auto&& item){ + return ctype.is_type(item.first, item.second); }) != ctypes.end(); + }; + + const auto ctype{mime_object().content_type()}; + if (!ctype) + return false; // no content-type: not an attachment. + + // we consider some parts _not_ to be attachments regardless of disposition + if (matches(*ctype,{{"application", "pgp-keys"}})) + return false; + + // we consider some parts to be attachments regardless of disposition + if (matches(*ctype,{{"image", "*"}, + {"audio", "*"}, + {"application", "*"}, + {"application", "x-patch"}})) + return true; + + // otherwise, rely on the disposition + return is_attachment(); +} + + + +#ifdef BUILD_TESTS +#include "utils/mu-test-utils.hh" + +static void +test_cooked_full() +{ + std::array<std::pair<std::string, std::string>, 4> cases = {{ + { "/hello/world/foo", "foo" }, + { "foo:/\n/bar", "bar"}, + { "Aap Noot Mies", "Aap-Noot-Mies"}, + { "..", "-"} + }}; + + for (auto&& test: cases) + assert_equal(cook_full(test.first), test.second); +} + +static void +test_cooked_minimal() +{ + std::array<std::pair<std::string, std::string>, 4> cases = {{ + { "/hello/world/foo", "foo" }, + { "foo:/\n/bar", "bar"}, + { "Aap Noot Mies.doc", "Aap Noot Mies.doc"}, + { "..", "-"} + }}; + + for (auto&& test: cases) + assert_equal(cook_minimal(test.first), test.second); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/message/message-part/cooked-full", test_cooked_full); + g_test_add_func("/message/message-part/cooked-minimal", test_cooked_minimal); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-message-part.hh b/lib/message/mu-message-part.hh new file mode 100644 index 0000000..1d31e0e --- /dev/null +++ b/lib/message/mu-message-part.hh @@ -0,0 +1,167 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#ifndef MU_MESSAGE_PART_HH__ +#define MU_MESSAGE_PART_HH__ + +#include <string> +#include <memory> +#include <utils/mu-option.hh> +#include <utils/mu-result.hh> + +namespace Mu { + +class MimeObject; // forward declaration; don't want to include for build-time + // reasons. + +class MessagePart { +public: + /** + * Construct MessagePart from a MimeObject + * + * @param obj + */ + MessagePart(const MimeObject& obj); + + /** + * Copy CTOR + * + * @param other + */ + MessagePart(const MessagePart& other); + + /** + * DTOR + * + */ + ~MessagePart(); + + /** + * Get the underlying MimeObject; you need to include mu-mime-object.hh + * to do anything useful with it. + * + * @return reference to the mime-object + */ + const MimeObject& mime_object() const noexcept; + + /** + * Filename for the mime-part file. This is a "cooked" filename with + * unallowed characters removed. If there's no filename specified, + * construct one (such as in the case of a MimeMessagePart). + * + * @param minimal if true, only perform *minimal* cookiing, where we + * only remove forward-slashes. + * + * @see raw_filename() + * + * @return the name + */ + Option<std::string> cooked_filename(bool minimal=false) const noexcept; + + /** + * Name for the mime-part file, i.e., MimePart::filename + * + * @return the filename or Nothing if there is none + */ + Option<std::string> raw_filename() const noexcept; + + /** + * Mime-type for the mime-part (e.g. "text/plain") + * + * @return the mime-part or Nothing if there is none + */ + Option<std::string> mime_type() const noexcept; + + + /** + * Get the content description for this part, or Nothing + * + * @return the content description + */ + Option<std::string> content_description() const noexcept; + + /** + * Get the length of the (unencoded) MIME-part. + * + * @return the size + */ + size_t size() const noexcept; + + /** + * Does this part have an "attachment" disposition? Otherwise it is + * "inline". Note that does *not* map 1:1 to a message's HasAttachment + * flag (which uses looks_like_attachment()) + * + * @return true or false. + */ + bool is_attachment() const noexcept; + + + /** + * Does this part appear to be an attachment from an end-users point of + * view? This uses some heuristics to guess. Some parts for which + * is_attachment() is true may not "really" be attachments, and + * vice-versa + * + * @return true or false. + */ + bool looks_like_attachment() const noexcept; + + /** + * Is this part signed? + * + * @return true or false + */ + bool is_signed() const noexcept; + + + /** + * Is this part encrypted? + * + * @return true or false + */ + bool is_encrypted() const noexcept; + + + /** + * Write (decoded) mime-part contents to string + * + * @return a string or nothing if there is no contemt + */ + Option<std::string> to_string() const noexcept; + + /** + * Write (decoded) mime part to a file + * + * @param path path to file + * @param overwrite whether to possibly overwrite + * + * @return size of file or or an error. + */ + Result<size_t> to_file(const std::string& path, bool overwrite) const noexcept; + + struct Private; +private: + const std::unique_ptr<MimeObject> mime_obj; +}; + +} // namespace Mu + +#endif /* MU_MESSAGE_PART_HH__ */ diff --git a/lib/message/mu-message.cc b/lib/message/mu-message.cc new file mode 100644 index 0000000..6ddd1f3 --- /dev/null +++ b/lib/message/mu-message.cc @@ -0,0 +1,863 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "mu-message.hh" +#include "gmime/gmime-references.h" +#include "gmime/gmime-stream-mem.h" +#include "mu-maildir.hh" + +#include <array> +#include <string> +#include <regex> +#include <utils/mu-utils.hh> +#include <utils/mu-error.hh> +#include <utils/mu-option.hh> +#include <utils/mu-lang-detector.hh> + +#include <atomic> +#include <mutex> +#include <cstdlib> + +#include <glib.h> +#include <glib/gstdio.h> +#include <gmime/gmime.h> + +#include "gmime/gmime-message.h" +#include "mu-mime-object.hh" + +using namespace Mu; + +struct Message::Private { + Private(Message::Options options): + opts{options}, doc{doc_opts(opts)} {} + Private(Message::Options options, Xapian::Document&& xdoc): + opts{options}, doc{std::move(xdoc), doc_opts(opts)} {} + + Message::Options opts; + Document doc; + mutable Option<MimeMessage> mime_msg; + + Flags flags{}; + Option<std::string> mailing_list; + std::vector<Part> parts; + + ::time_t ctime{}; + + std::string cache_path; + /* + * we only need to index these, so we don't + * really need these copy if we re-arrange things + * a bit + */ + Option<std::string> body_txt; + Option<std::string> body_html; + Option<std::string> embedded; + + Option<std::string> language; /* body ISO language code */ + +private: + Document::Options doc_opts(Message::Options mopts) { + return any_of(opts & Message::Options::SupportNgrams) ? + Document::Options::SupportNgrams : + Document::Options::None; + } +}; + + +static void fill_document(Message::Private& priv); + +static Result<struct stat> +get_statbuf(const std::string& path, Message::Options opts = Message::Options::None) +{ + if (none_of(opts & Message::Options::AllowRelativePath) && + !g_path_is_absolute(path.c_str())) + return Err(Error::Code::File, "path '{}' is not absolute", path); + + if (::access(path.c_str(), R_OK) != 0) + return Err(Error::Code::File, "file @ '{}' is not readable", path); + + struct stat statbuf{}; + if (::stat(path.c_str(), &statbuf) < 0) + return Err(Error::Code::File, "cannot stat {}: {}", path, + g_strerror(errno)); + + if (!S_ISREG(statbuf.st_mode)) + return Err(Error::Code::File, "not a regular file: {}", path); + + return Ok(std::move(statbuf)); +} + + +Message::Message(const std::string& path, Message::Options opts): + priv_{std::make_unique<Private>(opts)} +{ + const auto statbuf{get_statbuf(path, opts)}; + if (!statbuf) + throw statbuf.error(); + + priv_->ctime = statbuf->st_ctime; + + init_gmime(); + if (auto msg{MimeMessage::make_from_file(path)}; !msg) + throw msg.error(); + else + priv_->mime_msg = std::move(msg.value()); + + auto xpath{to_string_opt_gchar(g_canonicalize_filename(path.c_str(), NULL))}; + if (xpath) + priv_->doc.add(Field::Id::Path, std::move(xpath.value())); + + priv_->doc.add(Field::Id::Size, static_cast<int64_t>(statbuf->st_size)); + + // rest of the fields + fill_document(*priv_); +} + +Message::Message(const std::string& text, const std::string& path, + Message::Options opts): + priv_{std::make_unique<Private>(opts)} +{ + if (text.empty()) + throw Error{Error::Code::InvalidArgument, "text must not be empty"}; + + if (!path.empty()) { + auto xpath{to_string_opt_gchar(g_canonicalize_filename(path.c_str(), {}))}; + if (xpath) + priv_->doc.add(Field::Id::Path, std::move(xpath.value())); + } + + priv_->ctime = ::time({}); + + priv_->doc.add(Field::Id::Size, static_cast<int64_t>(text.size())); + + init_gmime(); + if (auto msg{MimeMessage::make_from_text(text)}; !msg) + throw msg.error(); + else + priv_->mime_msg = std::move(msg.value()); + + fill_document(*priv_); +} + + +Message::Message(Message&& other) noexcept +{ + *this = std::move(other); +} + +Message& +Message::operator=(Message&& other) noexcept +{ + if (this != &other) + priv_ = std::move(other.priv_); + + return *this; +} + +Message::Message(Xapian::Document&& doc): + priv_{std::make_unique<Private>(Message::Options::None, std::move(doc))} +{} + + +Message::~Message() = default; + +const Mu::Document& +Message::document() const +{ + return priv_->doc; +} + +Message::Options +Message::options() const +{ + return priv_->opts; +} + +unsigned +Message::docid() const +{ + return priv_->doc.xapian_document().get_docid(); +} + + +const Mu::Sexp& +Message::sexp() const +{ + return priv_->doc.sexp(); +} + +Result<void> +Message::set_maildir(const std::string& maildir) +{ + /* sanity check a little bit */ + if (maildir.empty() || + maildir.at(0) != '/' || + (maildir.size() > 1 && maildir.at(maildir.length()-1) == '/')) + return Err(Error::Code::Message, + "'{}' is not a valid maildir", maildir.c_str()); + + const auto path{document().string_value(Field::Id::Path)}; + if (path == maildir || path.find(maildir) == std::string::npos) + return Err(Error::Code::Message, + "'{}' is not a valid maildir for message @ {}", + maildir, path); + + priv_->doc.remove(Field::Id::Maildir); + priv_->doc.add(Field::Id::Maildir, maildir); + + return Ok(); +} + +void +Message::set_flags(Flags flags) +{ + priv_->doc.remove(Field::Id::Flags); + priv_->doc.add(flags); +} + +bool +Message::load_mime_message(bool reload) const +{ + if (priv_->mime_msg && !reload) + return true; + + const auto path{document().string_value(Field::Id::Path)}; + if (auto mime_msg{MimeMessage::make_from_file(path)}; !mime_msg) { + mu_warning("failed to load '{}': {}", + path, mime_msg.error().what()); + return false; + } else { + priv_->mime_msg = std::move(mime_msg.value()); + fill_document(*priv_); + return true; + } +} + +void +Message::unload_mime_message() const +{ + priv_->mime_msg = Nothing; +} + +bool +Message::has_mime_message() const +{ + return !!priv_->mime_msg; +} + + +static Priority +get_priority(const MimeMessage& mime_msg) +{ + constexpr std::array<std::pair<std::string_view, Priority>, 10> + prio_alist = {{ + {"high", Priority::High}, + {"1", Priority::High}, + {"2", Priority::High}, + + {"normal", Priority::Normal}, + {"3", Priority::Normal}, + + {"low", Priority::Low}, + {"list", Priority::Low}, + {"bulk", Priority::Low}, + {"4", Priority::Low}, + {"5", Priority::Low} + }}; + + const auto opt_str = mime_msg.header("Precedence") + .disjunction(mime_msg.header("X-Priority")) + .disjunction(mime_msg.header("Importance")); + + if (!opt_str) + return Priority::Normal; + + const auto it = seq_find_if(prio_alist, [&](auto&& item) { + return g_ascii_strncasecmp(item.first.data(), opt_str->c_str(), + item.first.size()) == 0; }); + + return it == prio_alist.cend() ? Priority::Normal : it->second; +} + + +/* see: http://does-not-exist.org/mail-archives/mutt-dev/msg08249.html */ +static std::vector<std::string> +extract_tags(const MimeMessage& mime_msg) +{ + constexpr std::array<std::pair<const char*, char>, 3> tag_headers = {{ + {"X-Label", ' '}, {"X-Keywords", ','}, {"Keywords", ' '} + }}; + + std::vector<std::string> tags; + seq_for_each(tag_headers, [&](auto&& item) { + if (auto&& hdr = mime_msg.header(item.first); hdr) { + for (auto&& tagval : split(*hdr, item.second)) { + tagval.erase(0, tagval.find_first_not_of(' ')); + tagval.erase(tagval.find_last_not_of(' ')+1); + tags.emplace_back(std::move(tagval)); + } + } + }); + + return tags; +} + +static Option<std::string> +get_mailing_list(const MimeMessage& mime_msg) +{ + char *dechdr, *res; + const char *b, *e; + + const auto hdr{mime_msg.header("List-Id")}; + if (!hdr) { + /* some marketing messages don't have a List-Id, but _do_ have a + * List-Unsubscribe; if so, return an empty string here, so this + * message is still flagged as "MailingList" + */ + if (const auto lu = mime_msg.header("List-Unsubscribe"); !!lu) + return ""; + else + return Nothing; + } + + dechdr = g_mime_utils_header_decode_phrase(NULL, hdr->c_str()); + if (!dechdr) + return {}; + + e = NULL; + b = ::strchr(dechdr, '<'); + if (b) + e = strchr(b, '>'); + + if (b && e) + res = g_strndup(b + 1, e - b - 1); + else + res = g_strdup(dechdr); + + g_free(dechdr); + + return to_string_opt_gchar(std::move(res)); +} + +static void +append_text(Option<std::string>& str, Option<std::string>&& app) +{ + if (!str && app) + str = std::move(*app); + else if (str && app) + str.value() += app.value(); +} + +static void +accumulate_text(const MimePart& part, Message::Private& info, + const MimeContentType& ctype) +{ + if (!ctype.is_type("text", "*")) + return; /* not a text type */ + + if (part.is_attachment()) + append_text(info.embedded, part.to_string()); + else if (ctype.is_type("text", "plain")) + append_text(info.body_txt, part.to_string()); + else if (ctype.is_type("text", "html")) + append_text(info.body_html, part.to_string()); +} + + +static bool /* heuristic */ +looks_like_attachment(const MimeObject& parent, const MessagePart& mpart) +{ + if (parent) { /* crypto multipart children are not considered attachments */ + if (const auto parent_ctype{parent.content_type()}; parent_ctype) { + if (parent_ctype->is_type("multipart", "signed") || + parent_ctype->is_type("multipart", "encrypted")) + return false; + } + } + + return mpart.looks_like_attachment(); +} + + +static void +process_part(const MimeObject& parent, const MimePart& part, + Message::Private& info, const MessagePart& mpart) +{ + const auto ctype{part.content_type()}; + if (!ctype) + return; + + // flag as calendar, if not already + if (none_of(info.flags & Flags::Calendar) && + ctype->is_type("text", "calendar")) + info.flags |= Flags::Calendar; + + // flag as attachment, if not already. + if (none_of(info.flags & Flags::HasAttachment) && + looks_like_attachment(parent, mpart)) + info.flags |= Flags::HasAttachment; + + // if there are text parts, gather. + accumulate_text(part, info, *ctype); +} + + +static void +process_message_part(const MimeMessagePart& msg_part, + Message::Private& info) +{ + auto submsg{msg_part.get_message()}; + if (!submsg) + return; + + submsg->for_each([&](auto&& parent, auto&& child_obj) { + /* NOTE: we only handle one level; ideally, we'd apply the whole + parsing machinery recursively; so this a little crude. */ + if (!child_obj.is_part()) + return; + if (const auto ctype{child_obj.content_type()}; !ctype) + return; + else if (ctype->is_type("text", "plain")) + append_text(info.embedded, MimePart{child_obj}.to_string()); + else if (ctype->is_type("text", "html")) { + if (auto&& str{MimePart{child_obj}.to_string()}; str) + append_text(info.embedded, html_to_text(*str)); + } + }); +} + +static void +handle_object(const MimeObject& parent, + const MimeObject& obj, Message::Private& info); + + +static void +handle_encrypted(const MimeMultipartEncrypted& part, Message::Private& info) +{ + if (!any_of(info.opts & Message::Options::Decrypt)) { + /* just added to the list */ + info.parts.emplace_back(part); + return; + } + + const auto proto{part.content_type_parameter("protocol").value_or("unknown")}; + const auto ctx = MimeCryptoContext::make(proto); + if (!ctx) { + mu_warning("failed to create context for protocol <{}>", proto); + return; + } + + auto res{part.decrypt(*ctx)}; + if (!res) { + mu_warning("failed to decrypt: {}", res.error().what()); + return; + } + + if (res->first.is_multipart()) { + MimeMultipart{res->first}.for_each( + [&](auto&& parent, auto&& child_obj) { + handle_object(parent, child_obj, info); + }); + + } else + handle_object(part, res->first, info); +} + + +static void +handle_object(const MimeObject& parent, + const MimeObject& obj, Message::Private& info) +{ + /* if it's an encrypted part we should decrypt, recurse */ + if (obj.is_multipart_encrypted()) + handle_encrypted(MimeMultipartEncrypted{obj}, info); + else if (obj.is_part() || + obj.is_message_part() || + obj.is_multipart_signed() || + obj.is_multipart_encrypted()) + info.parts.emplace_back(obj); + + if (obj.is_part()) + process_part(parent, obj, info, info.parts.back()); + else if (obj.is_message_part()) + process_message_part(obj, info); + else if (obj.is_multipart_signed()) + info.flags |= Flags::Signed; + else if (obj.is_multipart_encrypted()) { + /* FIXME: An encrypted part might be signed at the same time. + * In that case the signed flag is lost. */ + info.flags |= Flags::Encrypted; + } else if (obj.is_mime_application_pkcs7_mime()) { + MimeApplicationPkcs7Mime smime(obj); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + // CompressedData, CertsOnly, Unknown + switch (smime.smime_type()) { + case Mu::MimeApplicationPkcs7Mime::SecureMimeType::SignedData: + info.flags |= Flags::Signed; + break; + case Mu::MimeApplicationPkcs7Mime::SecureMimeType::EnvelopedData: + info.flags |= Flags::Encrypted; + break; + default: + break; + } +#pragma GCC diagnostic pop + } +} + +/** + * This message -- recursively walk through message, and initialize some + * other values that depend on another. + * + * @param mime_msg + * @param path + * @param info + */ +static void +process_message(const MimeMessage& mime_msg, const std::string& path, + Message::Private& info) +{ + /* only have file-flags when there's a path. */ + if (!path.empty()) { + info.flags = flags_from_path(path).value_or(Flags::None); + /* pseudo-flag --> unread means either NEW or NOT SEEN, just + * for searching convenience */ + if (any_of(info.flags & Flags::New) || none_of(info.flags & Flags::Seen)) + info.flags |= Flags::Unread; + } + + // parts + mime_msg.for_each([&](auto&& parent, auto&& child_obj) { + handle_object(parent, child_obj, info); + }); + + // get the mailing here, and use it do update flags, too. + info.mailing_list = get_mailing_list(mime_msg); + if (info.mailing_list) + info.flags |= Flags::MailingList; + +#ifdef HAVE_CLD2 + /* language detection requires the cld2 lib */ + if (info.body_txt) { /* attempt to get the body-language */ + if (const auto lang{detect_language(info.body_txt.value())}; lang) { + info.language = lang->code; + mu_debug("detected language: {}", lang->code); + } else + mu_debug("could not detect language"); + } +#endif /*HAVE_CLD2*/ +} + +static Mu::Result<std::string> +calculate_sha256(const std::string& path) +{ + g_autoptr(GChecksum) checksum{g_checksum_new(G_CHECKSUM_SHA256)}; + + FILE *file{::fopen(path.c_str(), "r")}; + if (!file) + return Err(Error{Error::Code::File, "failed to open {}: {}", + path, ::strerror(errno)}); + + std::array<uint8_t, 4096> buf{}; + while (true) { + const auto n = ::fread(buf.data(), 1, buf.size(), file); + if (n == 0) + break; + g_checksum_update(checksum, buf.data(), n); + } + + bool has_err = ::ferror(file) != 0; + ::fclose(file); + + if (has_err) + return Err(Error{Error::Code::File, "failed to read {}", path}); + + return Ok(g_checksum_get_string(checksum)); +} + +/** + * Get a fake-message-id for a message without one. + * + * @param path message path + * + * @return a fake message-id + */ +static std::string +fake_message_id(const std::string& path) +{ + constexpr auto mu_suffix{"@mu.id"}; + + // not a very good message-id, only for testing. + if (path.empty() || ::access(path.c_str(), R_OK) != 0) + return mu_format("{:08x}{}", g_str_hash(path.c_str()), mu_suffix); + if (const auto sha256_res{calculate_sha256(path)}; !sha256_res) + return mu_format("{:08x}{}", g_str_hash(path.c_str()), mu_suffix); + else + return mu_format("{}{}", sha256_res.value(), mu_suffix); +} + +/* many of the doc.add(fields ....) automatically update the sexp-list as well; + * however, there are some _extra_ values in the sexp-list that are not + * based on a field. So we add them here. + */ + + +static void +doc_add_list_post(Document& doc, const MimeMessage& mime_msg) +{ + /* some mailing lists do not set the reply-to; see pull #1278. So for + * those cases, check the List-Post address and use that instead */ + + GMatchInfo* minfo; + GRegex* rx; + const auto list_post{mime_msg.header("List-Post")}; + if (!list_post) + return; + + rx = g_regex_new("<?mailto:([a-z0-9!@#$%&'*+-/=?^_`{|}~]+)>?", + G_REGEX_CASELESS, (GRegexMatchFlags)0, {}); + g_return_if_fail(rx); + + Contacts contacts; + if (g_regex_match(rx, list_post->c_str(), (GRegexMatchFlags)0, &minfo)) { + auto address = (char*)g_match_info_fetch(minfo, 1); + contacts.push_back(Contact(address)); + g_free(address); + } + + g_match_info_free(minfo); + g_regex_unref(rx); + + doc.add_extra_contacts(":list-post", contacts); +} + +static void +doc_add_reply_to(Document& doc, const MimeMessage& mime_msg) +{ + doc.add_extra_contacts(":reply-to", mime_msg.contacts(Contact::Type::ReplyTo)); +} + +static void +fill_document(Message::Private& priv) +{ + /* hunt & gather info from message tree */ + Document& doc{priv.doc}; + MimeMessage& mime_msg{priv.mime_msg.value()}; + + const auto path{doc.string_value(Field::Id::Path)}; + const auto refs{mime_msg.references()}; + const auto& raw_message_id = mime_msg.message_id(); + const auto message_id = raw_message_id.has_value() && !raw_message_id->empty() + ? *raw_message_id + : fake_message_id(path); + + process_message(mime_msg, path, priv); + + doc_add_list_post(doc, mime_msg); /* only in sexp */ + doc_add_reply_to(doc, mime_msg); /* only in sexp */ + + field_for_each([&](auto&& field) { + /* insist on explicitly handling each */ +#pragma GCC diagnostic push +#pragma GCC diagnostic error "-Wswitch" + switch(field.id) { + case Field::Id::Bcc: + doc.add(field.id, mime_msg.contacts(Contact::Type::Bcc)); + break; + case Field::Id::BodyText: + doc.add(field.id, priv.body_txt); + if (priv.body_html) + doc.add(field.id, html_to_text(*priv.body_html)); + break; + case Field::Id::Cc: + doc.add(field.id, mime_msg.contacts(Contact::Type::Cc)); + break; + case Field::Id::Changed: + doc.add(field.id, priv.ctime); + break; + case Field::Id::Date: + doc.add(field.id, mime_msg.date()); + break; + case Field::Id::EmbeddedText: + doc.add(field.id, priv.embedded); + break; + case Field::Id::File: + for (auto&& part: priv.parts) + doc.add(field.id, part.raw_filename()); + break; + case Field::Id::Flags: + doc.add(priv.flags); + break; + case Field::Id::From: + doc.add(field.id, mime_msg.contacts(Contact::Type::From)); + break; + case Field::Id::Language: + doc.add(field.id, priv.language); + break; + case Field::Id::Maildir: /* already */ + break; + case Field::Id::MailingList: + doc.add(field.id, priv.mailing_list); + break; + case Field::Id::MessageId: + doc.add(field.id, message_id); + break; + case Field::Id::MimeType: + for (auto&& part: priv.parts) + doc.add(field.id, part.mime_type()); + break; + case Field::Id::Path: /* already */ + break; + case Field::Id::Priority: + doc.add(get_priority(mime_msg)); + break; + case Field::Id::References: + if (!refs.empty()) + doc.add(field.id, refs); + break; + case Field::Id::Size: /* already */ + break; + case Field::Id::Subject: + doc.add(field.id, mime_msg.subject().map(remove_ctrl)); + break; + case Field::Id::Tags: + if (auto&& tags{extract_tags(mime_msg)}; !tags.empty()) + doc.add(field.id, tags); + break; + case Field::Id::ThreadId: + // either the oldest reference, or otherwise the message id + doc.add(field.id, refs.empty() ? message_id : refs.at(0)); + break; + case Field::Id::To: + doc.add(field.id, mime_msg.contacts(Contact::Type::To)); + break; + /* LCOV_EXCL_START */ + case Field::Id::_count_: + default: + break; + /* LCOV_EXCL_STOP */ + } +#pragma GCC diagnostic pop + + }); +} + +Option<std::string> +Message::header(const std::string& header_field) const +{ + load_mime_message(); + return priv_->mime_msg->header(header_field); +} + +Option<std::string> +Message::body_text() const +{ + load_mime_message(); + return priv_->body_txt; +} + +Option<std::string> +Message::body_html() const +{ + load_mime_message(); + return priv_->body_html; +} + +Contacts +Message::all_contacts() const +{ + Contacts contacts; + + if (!load_mime_message()) + return contacts; /* empty */ + + return priv_->mime_msg->contacts(Contact::Type::None); /* get all types */ +} + +const std::vector<Message::Part>& +Message::parts() const +{ + if (!load_mime_message()) { + static std::vector<Message::Part> empty; + return empty; + } + + return priv_->parts; +} + +Result<std::string> +Message::cache_path(Option<size_t> index) const +{ + /* create tmpdir for this message, if needed */ + if (priv_->cache_path.empty()) { + GError *err{}; + auto tpath{to_string_opt_gchar(g_dir_make_tmp("mu-cache-XXXXXX", &err))}; + if (!tpath) + return Err(Error::Code::File, &err, "failed to create temp dir"); + + priv_->cache_path = std::move(tpath.value()); + } + + if (index) { + GError *err{}; + auto tpath = mu_format("{}/{}", priv_->cache_path, *index); + if (g_mkdir(tpath.c_str(), 0700) != 0) + return Err(Error::Code::File, &err, + "failed to create cache dir '{}'; err={}", tpath, errno); + return Ok(std::move(tpath)); + } else + + return Ok(std::string{priv_->cache_path}); +} + +// for now this only remove stray '/' at the end +std::string +Message::sanitize_maildir(const std::string& mdir) +{ + if (mdir.size() > 1 && mdir.at(mdir.length()-1) == '/') + return mdir.substr(0, mdir.length() - 1); + else + return mdir; +} + +Result<void> +Message::update_after_move(const std::string& new_path, + const std::string& new_maildir, + Flags new_flags) +{ + if (auto statbuf{get_statbuf(new_path)}; !statbuf) + return Err(statbuf.error()); + else + priv_->ctime = statbuf->st_ctime; + + priv_->doc.remove(Field::Id::Path); + priv_->doc.remove(Field::Id::Changed); + + priv_->doc.add(Field::Id::Path, new_path); + priv_->doc.add(Field::Id::Changed, priv_->ctime); + + set_flags(new_flags); + + if (const auto res = set_maildir(sanitize_maildir(new_maildir)); !res) + return res; + + return Ok(); +} diff --git a/lib/message/mu-message.hh b/lib/message/mu-message.hh new file mode 100644 index 0000000..0f029f4 --- /dev/null +++ b/lib/message/mu-message.hh @@ -0,0 +1,478 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MESSAGE_HH__ +#define MU_MESSAGE_HH__ + +#include <memory> +#include <string> +#include <vector> +#include <iostream> + +#include "mu-xapian-db.hh" + +#include "mu-contact.hh" +#include "mu-priority.hh" +#include "mu-flags.hh" +#include "mu-fields.hh" +#include "mu-document.hh" +#include "mu-message-part.hh" +#include "mu-message-file.hh" + +#include "utils/mu-utils.hh" +#include "utils/mu-option.hh" +#include "utils/mu-result.hh" +#include "utils/mu-sexp.hh" + +namespace Mu { + +class Message { +public: + enum struct Options { + None = 0, /**< Defaults */ + Decrypt = 1 << 0, /**< Attempt to decrypt */ + RetrieveKeys = 1 << 1, /**< Auto-retrieve crypto keys (implies network + * access) */ + AllowRelativePath = 1 << 2, /**< Allow relative paths for filename + * in make_from_path */ + SupportNgrams = 1 << 3, /**< Support ngrams, as used in + * CJK and other languages. */ + }; + + /** + * Move CTOR + * + * @param some other message + */ + Message(Message&& other) noexcept; + + /** + * operator= + * + * @param other move some object object + * + * @return + */ + Message& operator=(Message&& other) noexcept; + + /** + * Construct a message based on a path + * + * @param path path to message + * @param opts options + * + * @return a message or an error + */ + static Result<Message> make_from_path(const std::string& path, + Options opts={}) try { + return Ok(Message{path,opts}); + } catch (Error& err) { + return Err(err); + } + /* LCOV_EXCL_START */ + catch (...) { + return Err(Mu::Error(Error::Code::Message, + "failed to create message from path")); + } + /* LCOV_EXCL_STOP */ + + /** + * Construct a message based on a Xapian::Document + * + * @param doc a Mu Document + * + * @return a message or an error + */ + static Result<Message> make_from_document(Xapian::Document&& doc) try { + return Ok(Message{std::move(doc)}); + } catch (Error& err) { + return Err(err); + } + /* LCOV_EXCL_START */ + catch (...) { + return Err(Mu::Error(Error::Code::Message, + "failed to create message from document")); + } + /* LCOV_EXCL_STOP */ + + /** + * Construct a message from a string. This is mostly useful for testing. + * + * @param text message text + * @param path path to message - optional; path does not have to exist. + * @param opts options + * + * @return a message or an error + */ + static Result<Message> make_from_text(const std::string& text, + const std::string& path={}, + Options opts={}) try { + return Ok(Message{text, path, opts}); + } catch (Error& err) { + return Err(err); + } + /* LCOV_EXCL_START */ + catch (...) { + return Err(Mu::Error(Error::Code::Message, + "failed to create message from text")); + } + /* LCOV_EXCL_STOP */ + + /** + * DTOR + */ + ~Message(); + + /** + * Get the document. + * + * + * @return document + */ + const Document& document() const; + + + /** + * The message options for this message + * + * @return message options + */ + Options options() const; + + + /** + * Get the document-id, or 0 if non-existent. + * + * @return document id + */ + unsigned docid() const; + + /** + * Get the file system path of this message + * + * @return the path of this Message or NULL in case of error. + * the returned string should *not* be modified or freed. + */ + std::string path() const { return document().string_value(Field::Id::Path); } + + /** + * Get the sender (From:) of this message + * + * @return the sender(s) of this Message + */ + Contacts from() const { return document().contacts_value(Field::Id::From); } + + /** + * Get the recipient(s) (To:) for this message + * + * @return recipients + */ + Contacts to() const { return document().contacts_value(Field::Id::To); } + + /** + * Get the recipient(s) (Cc:) for this message + * + * @return recipients + */ + Contacts cc() const { return document().contacts_value(Field::Id::Cc); } + + /** + * Get the recipient(s) (Bcc:) for this message + * + * @return recipients + */ + Contacts bcc() const { return document().contacts_value(Field::Id::Bcc); } + + /** + * Get the maildir this message resides in; i.e., if the path is + * ~/Maildir/foo/bar/cur/msg, the maildir would typically be foo/bar + * + * This is determined when _storing_ the message (which uses + * set_maildir()) + * + * @return the maildir requested or empty */ + std::string maildir() const { return document().string_value(Field::Id::Maildir); } + + /** + * Set the maildir for this message. This is for use by the _store_ when + * it has determined the maildir for this message from the message's path and + * the root-maildir known by the store. + * + * @param maildir the maildir for this message + * + * @return Ok() or some error if the maildir is invalid + */ + Result<void> set_maildir(const std::string& maildir); + + /** + * Clean up the maildir. This is for internal use, but exposed for testing. + * For now cleaned-up means "stray trailing / removed". + * + * @param maildir some maildir + * + * @return a cleaned-up version + */ + static std::string sanitize_maildir(const std::string& maildir); + + /** + * Get the subject of this message + * + * @return the subject of this Message + */ + std::string subject() const { return document().string_value(Field::Id::Subject); } + + /** + * Get the Message-Id of this message + * + * @return the Message-Id of this message (without the enclosing <>), or + * a fake message-id for messages that don't have them. + * + * For file-backed message, this fake message-id is based on a hash of the + * message contents. For non-file-backed (test) messages, some other value + * is concocted. + */ + std::string message_id() const { return document().string_value(Field::Id::MessageId);} + + /** + * get the mailing list for a message, i.e. the mailing-list + * identifier in the List-Id header. + * + * @return the mailing list id for this message (without the enclosing <>) + * or NULL in case of error or if there is none. + */ + std::string mailing_list() const { return document().string_value(Field::Id::MailingList);} + + /** + * get the message date/time (the Date: field) as time_t + * + * @return message date/time or 0 in case of error or if there + * is no such header. + */ + ::time_t date() const { + return static_cast<::time_t>(document().integer_value(Field::Id::Date)); + } + + /** + * get the last change-time this message. For path/document-based + * messages this corresponds with the ctime of the underlying file; for + * the text-based ones (as used for testing) it is the creation time. + * + * @return last-change time or 0 if unknown + */ + ::time_t changed() const { + return static_cast<::time_t>(document().integer_value(Field::Id::Changed)); + } + + /** + * get the flags for this message. + * + * @return the file/content flags + */ + Flags flags() const { return document().flags_value(); } + + + /** + * Update the flags for this message. This is useful for flags + * that can only be determined after the message has been created already, + * such as the 'personal' flag. + * + * @param flags new flags. + */ + void set_flags(Flags flags); + + /** + * get the message priority for this message. The X-Priority, X-MSMailPriority, + * Importance and Precedence header are checked, in that order. if no known or + * explicit priority is set, Priority::Id::Normal is assumed + * + * @return the message priority + */ + Priority priority() const { return document().priority_value(); } + + /** + * get the file size in bytes of this message + * + * @return the filesize + */ + size_t size() const { return static_cast<size_t>(document().integer_value(Field::Id::Size)); } + + /** + * Get the (possibly empty) list of references (consisting of both the + * References and In-Reply-To fields), with the oldest first and the + * direct parent as the last one. Note, any reference (message-id) will + * appear at most once, duplicates and fake-message-id (see impls) are + * filtered out. + * + * @return a vec with the references for this msg. + */ + std::vector<std::string> references() const { + return document().string_vec_value(Field::Id::References); + } + + /** + * Get the thread-id for this message. This is the message-id of the + * oldest-known (grand) parent, or the message-id of this message if + * none. + * + * @return the thread id. + */ + std::string thread_id() const { + return document().string_value(Field::Id::ThreadId); + } + + /** + * get the list of tags (ie., X-Label) + * + * @param msg a valid MuMsg + * + * @return a list with the tags for this msg. Don't modify/free + */ + std::vector<std::string> tags() const { + return document() + .string_vec_value(Field::Id::Tags); + } + + /* + * Convert to Sexp + */ + + /** + * Get the s-expression for this message. Stays valid as long as this + * message is. + * + * @return an Sexp representing the message. + */ + const Sexp& sexp() const; + + /* + * And some non-const message, for updating an existing + * message after a file-system move. + * + * @return Ok or an error. + */ + Result<void> update_after_move(const std::string& new_path, + const std::string& new_maildir, + Flags new_flags); + /* + * Below require a file-backed message, which is a relatively slow + * if there isn't one already; see load_mime_message() + */ + + /** + * Get the text body + * + * @return text body + */ + Option<std::string> body_text() const; + + /** + * Get the HTML body + * + * @return text body + */ + Option<std::string> body_html() const; + + /** + * Get some message-header + * + * @param header_field name of the header + * + * @return the value (UTF-8), or Nothing. + */ + Option<std::string> header(const std::string& header_field) const; + + + /** + * Get all contacts for this message. + * + * @return contacts + */ + Contacts all_contacts() const; + + /** + * Get information about MIME-parts in this message. + * + * @return mime-part info. + */ + using Part = MessagePart; + const std::vector<Part>& parts() const; + + /** + * Get the path to a cache directory for this message, which is useful + * for temporarily saving attachments + * + * @param index optionally, create <cache-path>/<index> instead; + * this is useful for having part-specific subdirectories. + * + * @return path to a (created) cache directory, or an error. + */ + Result<std::string> cache_path(Option<size_t> index={}) const; + + + /** + * Load the GMime (file) message (for a database-backed message), + * if not already (but see @param reload). + * + * Affects cached-state only, so we still mark this as 'const' + * + * @param reload whether to force reloading (even if already) + * + * @return true if loading worked; false otherwise. + */ + bool load_mime_message(bool reload=false) const; + + /** + * Clear the GMime message. + * + * Affects cached-state only, so we still mark this as 'const' + */ + void unload_mime_message() const; + + /** + * Has a (file-base) GMime message been loaded? + * + * + * @return true or false + */ + bool has_mime_message() const; + + struct Private; + + /* + * Usually the make_ builders are better to create a message, but in + * some special cases, we need a heap-allocated message... */ + + Message(Xapian::Document&& xdoc); + Message(const std::string& path, Options opts); + +private: + Message(const std::string& str, const std::string& path, Options opt); + + std::unique_ptr<Private> priv_; + +}; // Message +MU_ENABLE_BITOPS(Message::Options); + +static inline auto +format_as(const Message& msg) { + return msg.path(); +} + +} // Mu +#endif /* MU_MESSAGE_HH__ */ diff --git a/lib/message/mu-mime-object.cc b/lib/message/mu-mime-object.cc new file mode 100644 index 0000000..a75da5b --- /dev/null +++ b/lib/message/mu-mime-object.cc @@ -0,0 +1,798 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#include "mu-mime-object.hh" +#include "gmime/gmime-message.h" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include <mutex> +#include <regex> +#include <fcntl.h> +#include <sys/stat.h> +#include <errno.h> + +using namespace Mu; + + + +/* note, we do the gmime initialization here rather than in mu-runtime, because this way + * we don't need mu-runtime for simple cases -- such as our unit tests. Also note that we + * need gmime init even for the doc backend, as we use the address parsing functions also + * there. */ + +void +Mu::init_gmime(void) +{ + // fast path. + static bool gmime_initialized = false; + if (gmime_initialized) + return; + + static std::mutex gmime_lock; + std::lock_guard lock (gmime_lock); + if (gmime_initialized) + return; // already + + mu_debug("initializing gmime {}.{}.{}", + gmime_major_version, + gmime_minor_version, + gmime_micro_version); + + g_mime_init(); + gmime_initialized = true; + + std::atexit([] { + mu_debug("shutting down gmime"); + g_mime_shutdown(); + gmime_initialized = false; + }); +} + + +std::string +Mu::address_rfc2047(const Contact& contact) +{ + init_gmime(); + + InternetAddress *addr = + internet_address_mailbox_new(contact.name.c_str(), + contact.email.c_str()); + + std::string encoded = to_string_gchar( + internet_address_to_string(addr, {}, true)); + + g_object_unref(addr); + + return encoded; +} + + +/* + * MimeObject + */ + +Option<std::string> +MimeObject::header(const std::string& hdr) const noexcept +{ + if (auto val{g_mime_object_get_header(self(), hdr.c_str())}; !val) + return Nothing; + else if (!g_utf8_validate(val, -1, {})) + return utf8_clean(val); + else + return std::string{val}; +} + + +std::vector<std::pair<std::string, std::string>> +MimeObject::headers() const noexcept +{ + GMimeHeaderList *lst; + + lst = g_mime_object_get_header_list(self()); /* _not_ owned */ + if (!lst) + return {}; + + std::vector<std::pair<std::string, std::string>> hdrs; + const auto hdr_num{g_mime_header_list_get_count(lst)}; + + for (int i = 0; i != hdr_num; ++i) { + GMimeHeader *hdr{g_mime_header_list_get_header_at(lst, i)}; + if (!hdr) /* ^^^ _not_ owned */ + continue; + const auto name{g_mime_header_get_name(hdr)}; + const auto val{g_mime_header_get_value(hdr)}; + if (!name || !val) + continue; + hdrs.emplace_back(name, val); + } + + return hdrs; +} + +Result<size_t> +MimeObject::write_to_stream(const MimeFormatOptions& f_opts, + MimeStream& stream) const +{ + auto written = g_mime_object_write_to_stream(self(), f_opts.get(), + GMIME_STREAM(stream.object())); + if (written < 0) + return Err(Error::Code::File, "failed to write mime-object to stream"); + else + return Ok(static_cast<size_t>(written)); +} + +Result<size_t> +MimeObject::to_file(const std::string& path, bool overwrite) const noexcept +{ + GError *err{}; + auto strm{g_mime_stream_fs_open(path.c_str(), + O_WRONLY | O_CREAT | O_TRUNC |(overwrite ? 0 : O_EXCL), + S_IRUSR|S_IWUSR, + &err)}; + if (!strm) + return Err(Error::Code::File, &err, "failed to open '{}'", path); + + MimeStream stream{MimeStream::make_from_stream(strm)}; + return write_to_stream({}, stream); +} + + +Option<std::string> +MimeObject::to_string_opt() const noexcept +{ + auto stream{MimeStream::make_mem()}; + if (!stream) { + mu_warning("failed to create mem stream"); + return Nothing; + } + + const auto written = g_mime_object_write_to_stream( + self(), {}, GMIME_STREAM(stream.object())); + if (written < 0) { + mu_warning("failed to write object to stream"); + return Nothing; + } + + std::string buffer; + buffer.resize(written + 1); + stream.reset(); + + auto bytes{g_mime_stream_read(GMIME_STREAM(stream.object()), + buffer.data(), written)}; + if (bytes < 0) + return Nothing; + + buffer.data()[written]='\0'; + buffer.resize(written); + + return buffer; +} + + +/* + * MimeCryptoContext + */ + +Result<size_t> +MimeCryptoContext::import_keys(MimeStream& stream) +{ + GError *err{}; + auto res = g_mime_crypto_context_import_keys( + self(), GMIME_STREAM(stream.object()), &err); + + if (res < 0) + return Err(Error::Code::File, &err, + "error importing keys"); + + return Ok(static_cast<size_t>(res)); +} + +void +MimeCryptoContext::set_request_password(PasswordRequestFunc pw_func) +{ + static auto request_func = pw_func; + + g_mime_crypto_context_set_request_password( + self(), + [](GMimeCryptoContext *ctx, + const char *user_id, + const char *prompt, + gboolean reprompt, + GMimeStream *response, + GError **err) -> gboolean { + MimeStream mstream{MimeStream::make_from_stream(response)}; + + auto res = request_func(MimeCryptoContext(ctx), + std::string{user_id ? user_id : ""}, + std::string{prompt ? prompt : ""}, + !!reprompt, + mstream); + if (res) + return TRUE; + + res.error().fill_g_error(err); + return FALSE; + }); + +} + +Result<void> +MimeCryptoContext::setup_gpg_test(const std::string& testpath) +{ + /* setup clean environment for testing; inspired by gmime */ + + g_setenv ("GNUPGHOME", join_paths(testpath, ".gnupg").c_str(), 1); + + /* disable environment variables that gpg-agent uses for pinentry */ + g_unsetenv ("DBUS_SESSION_BUS_ADDRESS"); + g_unsetenv ("DISPLAY"); + g_unsetenv ("GPG_TTY"); + + if (g_mkdir_with_parents((testpath + "/.gnupg").c_str(), 0700) != 0) + return Err(Error::Code::File, + "failed to create gnupg dir; err={}", errno); + + auto write_gpgfile=[&](const std::string& fname, const std::string& data) + -> Result<void> { + + GError *err{}; + std::string path{mu_format("{}/{}", testpath, fname)}; + if (!g_file_set_contents(path.c_str(), data.c_str(), data.size(), &err)) + return Err(Error::Code::File, &err, "failed to write {}", path); + else + return Ok(); + }; + + // some more elegant way? + if (auto&& res = write_gpgfile("gpg.conf", "pinentry-mode loopback\n"); !res) + return res; + if (auto&& res = write_gpgfile("gpgsm.conf", "disable-crl-checks\n")) + return res; + + return Ok(); +} + + +/* + * MimeMessage + */ + + + +static Result<MimeMessage> +make_from_stream(GMimeStream* &&stream/*consume*/) +{ + init_gmime(); + GMimeParser *parser{g_mime_parser_new_with_stream(stream)}; + g_object_unref(stream); + if (!parser) + return Err(Error::Code::Message, "cannot create mime parser"); + + GMimeMessage *gmime_msg{g_mime_parser_construct_message(parser, NULL)}; + g_object_unref(parser); + if (!gmime_msg) + return Err(Error::Code::Message, "message seems invalid"); + + auto mime_msg{MimeMessage{std::move(G_OBJECT(gmime_msg))}}; + g_object_unref(gmime_msg); + + return Ok(std::move(mime_msg)); +} + +Result<MimeMessage> +MimeMessage::make_from_file(const std::string& path) +{ + GError* err{}; + init_gmime(); + if (auto&& stream{g_mime_stream_file_open(path.c_str(), "r", &err)}; !stream) + return Err(Error::Code::Message, &err, + "failed to open stream for {}", path); + else + return make_from_stream(std::move(stream)); +} + +Result<MimeMessage> +MimeMessage::make_from_text(const std::string& text) +{ + init_gmime(); + if (auto&& stream{g_mime_stream_mem_new_with_buffer( + text.c_str(), text.length())}; !stream) + return Err(Error::Code::Message, + "failed to open stream for string"); + else + return make_from_stream(std::move(stream)); +} + +Option<int64_t> +MimeMessage::date() const noexcept +{ + GDateTime *dt{g_mime_message_get_date(self())}; + if (!dt) + return Nothing; + else + return g_date_time_to_unix(dt); +} + +constexpr Option<GMimeAddressType> +address_type(Contact::Type ctype) +{ + switch(ctype) { + case Contact::Type::Bcc: + return GMIME_ADDRESS_TYPE_BCC; + case Contact::Type::Cc: + return GMIME_ADDRESS_TYPE_CC; + case Contact::Type::From: + return GMIME_ADDRESS_TYPE_FROM; + case Contact::Type::To: + return GMIME_ADDRESS_TYPE_TO; + case Contact::Type::ReplyTo: + return GMIME_ADDRESS_TYPE_REPLY_TO; + case Contact::Type::Sender: + return GMIME_ADDRESS_TYPE_SENDER; + case Contact::Type::None: + default: + return Nothing; + } +} + +static Mu::Contacts +all_contacts(const MimeMessage& msg) +{ + Contacts contacts; + + for (auto&& cctype: { + Contact::Type::Sender, + Contact::Type::From, + Contact::Type::ReplyTo, + Contact::Type::To, + Contact::Type::Cc, + Contact::Type::Bcc + }) { + auto addrs{msg.contacts(cctype)}; + std::move(addrs.begin(), addrs.end(), + std::back_inserter(contacts)); + } + + return contacts; +} + +Mu::Contacts +MimeMessage::contacts(Contact::Type ctype) const noexcept +{ + /* special case: get all */ + if (ctype == Contact::Type::None) + return all_contacts(*this); + + const auto atype{address_type(ctype)}; + if (!atype) + return {}; + + auto addrs{g_mime_message_get_addresses(self(), *atype)}; + if (!addrs) + return {}; + + const auto msgtime{date().value_or(0)}; + + Contacts contacts; + auto lst_len{internet_address_list_length(addrs)}; + contacts.reserve(lst_len); + for (auto i = 0; i != lst_len; ++i) { + + auto&& addr{internet_address_list_get_address(addrs, i)}; + const auto name{internet_address_get_name(addr)}; + + if (G_UNLIKELY(!INTERNET_ADDRESS_IS_MAILBOX(addr))) + continue; + + const auto email{internet_address_mailbox_get_addr ( + INTERNET_ADDRESS_MAILBOX(addr))}; + if (G_UNLIKELY(!email)) + continue; + + contacts.emplace_back(email, name ? name : "", ctype, msgtime); + } + + return contacts; +} + +/* + * references() returns the concatenation of the References and In-Reply-To + * message-ids (in that order). Duplicates are removed. + * + * The _first_ one in the list determines the thread-id for the message. + */ +std::vector<std::string> +MimeMessage::references() const noexcept +{ + // is ref already in the list? O(n) but with small n. + auto is_dup = [](auto&& seq, const std::string& ref) { + return seq_some(seq, [&](auto&& str) { return ref == str; }); + }; + + auto is_fake = [](auto&& msgid) { + // this is bit ugly; protonmail injects fake References which + // can otherwise screw up threading. + if (g_str_has_suffix(msgid, "protonmail.internalid")) + return true; + /* ... */ + return false; + }; + + std::vector<std::string> refs; + for (auto&& ref_header: { "References", "In-reply-to" }) { + + auto hdr{header(ref_header)}; + if (!hdr) + continue; + + GMimeReferences *mime_refs{g_mime_references_parse({}, hdr->c_str())}; + refs.reserve(refs.size() + g_mime_references_length(mime_refs)); + + for (auto i = 0; i != g_mime_references_length(mime_refs); ++i) { + const auto msgid{g_mime_references_get_message_id(mime_refs, i)}; + if (msgid && !is_dup(refs, msgid) && !is_fake(msgid)) + refs.emplace_back(msgid); + } + g_mime_references_free(mime_refs); + } + + return refs; +} + +void +MimeMessage::for_each(const ForEachFunc& func) const noexcept +{ + struct CallbackData { const ForEachFunc& func; }; + CallbackData cbd{func}; + + g_mime_message_foreach( + self(), + [] (GMimeObject *parent, GMimeObject *part, gpointer user_data) { + auto cb_data{reinterpret_cast<CallbackData*>(user_data)}; + cb_data->func(MimeObject{parent}, MimeObject{part}); + }, &cbd); +} + + + +/* + * MimePart + */ +size_t +MimePart::size() const noexcept +{ + auto wrapper{g_mime_part_get_content(self())}; + if (!wrapper) { + mu_warning("failed to get content wrapper"); + return 0; + } + + auto stream{g_mime_data_wrapper_get_stream(wrapper)}; + if (!stream) { + mu_warning("failed to get stream"); + return 0; + } + + return static_cast<size_t>(g_mime_stream_length(stream)); +} +Option<std::string> +MimePart::to_string() const noexcept +{ + /* + * easy case: text. this automatically handles conversion to utf-8. + */ + if (GMIME_IS_TEXT_PART(self())) { + if (char* txt{g_mime_text_part_get_text(GMIME_TEXT_PART(self()))}; !txt) + return Nothing; + else + return to_string_gchar(std::move(txt)/*consumes*/); + } + + /* + * harder case: read from stream manually + */ + GMimeDataWrapper *wrapper{g_mime_part_get_content(self())}; + if (!wrapper) { /* this happens with invalid mails */ + mu_warning("failed to create data wrapper"); + return Nothing; + } + + GMimeStream *stream{g_mime_stream_mem_new()}; + if (!stream) { + mu_warning("failed to create mem stream"); + return Nothing; + } + + ssize_t buflen{g_mime_data_wrapper_write_to_stream(wrapper, stream)}; + if (buflen <= 0) { /* empty buffer, not an error */ + g_object_unref(stream); + return Nothing; + } + + std::string buffer; + buffer.resize(buflen + 1); + g_mime_stream_reset(stream); + + auto bytes{g_mime_stream_read(stream, buffer.data(), buflen)}; + g_object_unref(stream); + if (bytes < 0) + return Nothing; + + buffer.resize(bytes + 1); + + return buffer; +} + +Result<size_t> +MimePart::to_file(const std::string& path, bool overwrite) const noexcept +{ + MimeDataWrapper wrapper{g_mime_part_get_content(self())}; + if (!wrapper) /* this happens with invalid mails */ + return Err(Error::Code::File, "failed to create data wrapper"); + + GError *err{}; + auto strm{g_mime_stream_fs_open(path.c_str(), + O_WRONLY | O_CREAT | O_TRUNC |(overwrite ? 0 : O_EXCL), + S_IRUSR|S_IWUSR, + &err)}; + if (!strm) + return Err(Error::Code::File, &err, "failed to open '{}'", path); + + MimeStream stream{MimeStream::make_from_stream(strm)}; + ssize_t written{g_mime_data_wrapper_write_to_stream( + GMIME_DATA_WRAPPER(wrapper.object()), + GMIME_STREAM(stream.object()))}; + + if (written < 0) { + return Err(Error::Code::File, &err, + "failed to write to '{}'", path); + } + + return Ok(static_cast<size_t>(written)); +} + +void +MimeMultipart::for_each(const ForEachFunc& func) const noexcept +{ + struct CallbackData { const ForEachFunc& func; }; + CallbackData cbd{func}; + + g_mime_multipart_foreach( + self(), + [] (GMimeObject *parent, GMimeObject *part, gpointer user_data) { + auto cb_data{reinterpret_cast<CallbackData*>(user_data)}; + cb_data->func(MimeObject{parent}, MimeObject{part}); + }, &cbd); +} + + +/* + * we need to be able to pass a crypto-context to the verify(), but + * g_mime_multipart_signed_verify() doesn't offer that anymore in GMime 3.x. + * + * So, add that by reimplementing it a bit (follow the upstream impl) + */ + + +static bool +mime_types_equal (const std::string& mime_type, const std::string& official_type) +{ + if (g_ascii_strcasecmp(mime_type.c_str(), official_type.c_str()) == 0) + return true; + + const auto slash_pos = official_type.find("/"); + if (slash_pos == std::string::npos || slash_pos == 0) + return false; + + /* If the official mime-type's subtype already begins with "x-", then there's + * nothing else to check. */ + const auto subtype{official_type.substr(slash_pos + 1)}; + if (g_ascii_strncasecmp (subtype.c_str(), "x-", 2) == 0) + return false; + const auto supertype{official_type.substr(0, slash_pos - 1)}; + const auto xtype{official_type.substr(0, slash_pos - 1) + "x-" + subtype}; + + /* Check if the "x-" version of the official mime-type matches the + * supplied mime-type. For example, if the official mime-type is + * "application/pkcs7-signature", then we also want to match + * "application/x-pkcs7-signature". */ + return g_ascii_strcasecmp(mime_type.c_str(), xtype.c_str()) == 0; +} + + +/** + * A bit of a monster, this impl. + * + * It's the transliteration of the g_mime_multipart_signed_verify() which + * adds the feature of passing in the CryptoContext. + * + */ +Result<std::vector<MimeSignature>> +MimeMultipartSigned::verify(const MimeCryptoContext& ctx, VerifyFlags vflags) const noexcept +{ + if (g_mime_multipart_get_count(GMIME_MULTIPART(self())) < 2) + return Err(Error::Code::Crypto, "cannot verify, not enough subparts"); + + const auto proto{content_type_parameter("protocol")}; + const auto sign_proto{ctx.signature_protocol()}; + + if (!proto || !sign_proto || !mime_types_equal(*proto, *sign_proto)) + return Err(Error::Code::Crypto, "unsupported protocol {}", + proto.value_or("<unknown>")); + + const auto sig{signed_signature_part()}; + const auto content{signed_content_part()}; + if (!sig || !content) + return Err(Error::Code::Crypto, "cannot find part"); + + const auto sig_mime_type{sig->mime_type()}; + if (!sig || !mime_types_equal(sig_mime_type.value_or("<none>"), *sign_proto)) + return Err(Error::Code::Crypto, "failed to find matching signature part"); + + MimeFormatOptions fopts{g_mime_format_options_new()}; + g_mime_format_options_set_newline_format(fopts.get(), GMIME_NEWLINE_FORMAT_DOS); + + MimeStream stream{MimeStream::make_mem()}; + if (auto&& res = content->write_to_stream(fopts, stream); !res) + return Err(res.error()); + stream.reset(); + + MimeDataWrapper wrapper{g_mime_part_get_content(GMIME_PART(sig->object()))}; + MimeStream sigstream{MimeStream::make_mem()}; + if (auto&& res = wrapper.write_to_stream(sigstream); !res) + return Err(res.error()); + sigstream.reset(); + + GError *err{}; + GMimeSignatureList *siglist{g_mime_crypto_context_verify( + GMIME_CRYPTO_CONTEXT(ctx.object()), + static_cast<GMimeVerifyFlags>(vflags), + GMIME_STREAM(stream.object()), + GMIME_STREAM(sigstream.object()), + {}, + &err)}; + if (!siglist) + return Err(Error::Code::Crypto, &err, "failed to verify"); + + std::vector<MimeSignature> sigs; + for (auto i = 0; + i != g_mime_signature_list_length(siglist); ++i) { + GMimeSignature *msig = g_mime_signature_list_get_signature(siglist, i); + sigs.emplace_back(MimeSignature(msig)); + } + g_object_unref(siglist); + + return sigs; +} + + +std::vector<MimeCertificate> +MimeDecryptResult::recipients() const noexcept +{ + GMimeCertificateList *lst{g_mime_decrypt_result_get_recipients(self())}; + if (!lst) + return {}; + + std::vector<MimeCertificate> certs; + for (int i = 0; i != g_mime_certificate_list_length(lst); ++i) + certs.emplace_back( + MimeCertificate( + g_mime_certificate_list_get_certificate(lst, i))); + + return certs; +} + +std::vector<MimeSignature> +MimeDecryptResult::signatures() const noexcept +{ + GMimeSignatureList *lst{g_mime_decrypt_result_get_signatures(self())}; + if (!lst) + return {}; + + std::vector<MimeSignature> sigs; + for (auto i = 0; i != g_mime_signature_list_length(lst); ++i) { + GMimeSignature *sig = g_mime_signature_list_get_signature(lst, i); + sigs.emplace_back(MimeSignature(sig)); + } + + return sigs; +} +/** + * Like verify, a bit of a monster, this impl. + * + * It's the transliteration of the g_mime_multipart_encrypted_decrypt() which + * adds the feature of passing in the CryptoContext. + * + */ + +Mu::Result<MimeMultipartEncrypted::Decrypted> +MimeMultipartEncrypted::decrypt(const MimeCryptoContext& ctx, DecryptFlags dflags, + const std::string& session_key) const noexcept +{ + if (g_mime_multipart_get_count(GMIME_MULTIPART(self())) < 2) + return Err(Error::Code::Crypto, "cannot decrypted, not enough subparts"); + + const auto proto{content_type_parameter("protocol")}; + const auto enc_proto{ctx.encryption_protocol()}; + + if (!proto || !enc_proto || !mime_types_equal(*proto, *enc_proto)) + return Err(Error::Code::Crypto, "unsupported protocol {}", + proto.value_or("<unknown>")); + + const auto version{encrypted_version_part()}; + const auto encrypted{encrypted_content_part()}; + if (!version || !encrypted) + return Err(Error::Code::Crypto, "cannot find part"); + + if (!mime_types_equal(version->mime_type().value_or(""), proto.value())) + return Err(Error::Code::Crypto, + "cannot decrypt; unexpected version content-type '{}' != '{}'", + version->mime_type().value_or(""), proto.value()); + + if (!mime_types_equal(encrypted->mime_type().value_or(""), + "application/octet-stream")) + return Err(Error::Code::Crypto, + "cannot decrypt; unexpected encrypted content-type '{}'", + encrypted->mime_type().value_or("")); + + const auto content{encrypted->content()}; + auto ciphertext{MimeStream::make_mem()}; + content.write_to_stream(ciphertext); + ciphertext.reset(); + + auto stream{MimeStream::make_mem()}; + auto filtered{MimeStream::make_filtered(stream)}; + auto filter{g_mime_filter_dos2unix_new(FALSE)}; + g_mime_stream_filter_add(GMIME_STREAM_FILTER(filtered.object()), + filter); + g_object_unref(filter); + + GError *err{}; + GMimeDecryptResult *dres = + g_mime_crypto_context_decrypt(GMIME_CRYPTO_CONTEXT(ctx.object()), + static_cast<GMimeDecryptFlags>(dflags), + session_key.empty() ? + NULL : session_key.c_str(), + GMIME_STREAM(ciphertext.object()), + GMIME_STREAM(filtered.object()), + &err); + if (!dres) + return Err(Error::Code::Crypto, &err, "decryption failed"); + + filtered.flush(); + stream.reset(); + + auto parser{g_mime_parser_new()}; + g_mime_parser_init_with_stream(parser, GMIME_STREAM(stream.object())); + + auto decrypted{g_mime_parser_construct_part(parser, NULL)}; + g_object_unref(parser); + if (!decrypted) { + g_object_unref(dres); + return Err(Error::Code::Crypto, "failed to parse decrypted part"); + } + + Decrypted result = { MimeObject{decrypted}, MimeDecryptResult{dres} }; + + g_object_unref(decrypted); + g_object_unref(dres); + + return Ok(std::move(result)); +} diff --git a/lib/message/mu-mime-object.hh b/lib/message/mu-mime-object.hh new file mode 100644 index 0000000..bfb2867 --- /dev/null +++ b/lib/message/mu-mime-object.hh @@ -0,0 +1,1389 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MIME_OBJECT_HH__ +#define MU_MIME_OBJECT_HH__ + +#include <stdexcept> +#include <string> +#include <functional> +#include <array> +#include <vector> +#include <gmime/gmime.h> +#include "gmime/gmime-application-pkcs7-mime.h" +#include "gmime/gmime-crypto-context.h" +#include "utils/mu-option.hh" +#include "utils/mu-result.hh" +#include "utils/mu-utils.hh" +#include "mu-contact.hh" + +namespace Mu { + +/* non-GObject types */ + +using MimeFormatOptions = deletable_unique_ptr<GMimeFormatOptions, g_mime_format_options_free>; + +/** + * Initialize gmime (idempotent) + * + */ +void init_gmime(void); + + +/** + * Get a RFC2047-compatible address for the given contact + * + * @param contact a contact + * + * @return an address string + */ +std::string address_rfc2047(const Contact& contact); + +class Object { +public: + /** + * Default CTOR + * + */ + Object() noexcept: self_{} {} + + /** + * Create an object from a GObject + * + * @param obj a gobject. A ref is added. + */ + Object(GObject* &&obj): self_{G_OBJECT(g_object_ref(obj))} { + if (!G_IS_OBJECT(obj)) + throw std::runtime_error("not a g-object"); + } + + /** + * Copy CTOR + * + * @param other some other Object + */ + Object(const Object& other) noexcept { *this = other; } + + /** + * Move CTOR + * + * @param other some other Object + */ + Object(Object&& other) noexcept { *this = std::move(other); } + + /** + * operator= + * + * @param other copy some other object + * + * @return *this + */ + Object& operator=(const Object& other) noexcept { + + if (this != &other) { + auto oldself = self_; + self_ = other.self_ ? + G_OBJECT(g_object_ref(other.self_)) : nullptr; + if (oldself) + g_object_unref(oldself); + } + return *this; + } + + /** + * operator= + * + * @param other move some object object + * + * @return + */ + Object& operator=(Object&& other) noexcept { + + if (this != &other) { auto oldself = self_; + self_ = other.self_; + other.self_ = nullptr; + if (oldself) + g_object_unref(oldself); + } + return *this; + } + + /** + * DTOR + */ + virtual ~Object() { + if (self_) { + g_object_unref(self_); + } + } + + /** + * operator bool + * + * @return true if object wraps a GObject, false otherwise + */ + operator bool() const noexcept { return !!self_; } + + /** + * Get a ptr to the underlying GObject + * + * @return GObject or NULL + */ + GObject* object() const { return self_; } + + + /** + * Unref the object + * + */ + void unref() noexcept { + g_object_unref(self_); + } + + + /** + * Ref the object + * + */ + void ref() noexcept { + g_object_ref(self_); + } + + +private: + mutable GObject *self_{}; +}; + + + + + + +/** + * Thin wrapper around a GMimeContentType + * + */ +struct MimeContentType: public Object { + + MimeContentType(GMimeContentType *ctype) : Object{G_OBJECT(ctype)} { + if (!GMIME_IS_CONTENT_TYPE(self())) + throw std::runtime_error("not a content-type"); + } + std::string media_type() const noexcept { + return g_mime_content_type_get_media_type(self()); + } + std::string media_subtype() const noexcept { + return g_mime_content_type_get_media_subtype(self()); + } + + Option<std::string> mime_type() const noexcept { + return to_string_opt_gchar(g_mime_content_type_get_mime_type(self())); + } + + bool is_type(const std::string& type, const std::string& subtype) const { + return g_mime_content_type_is_type(self(), type.c_str(), + subtype.c_str()); + } +private: + GMimeContentType* self() const { + return reinterpret_cast<GMimeContentType*>(object()); + } +}; + + + + + +/** + * Thin wrapper around a GMimeStream + * + */ +struct MimeStream: public Object { + + ssize_t write(const char* buf, ::size_t size) { + return g_mime_stream_write(self(), buf, size); + } + + bool reset() { + return g_mime_stream_reset(self()) < 0 ? false : true; + } + + bool flush() { + return g_mime_stream_flush(self()) < 0 ? false : true; + } + + static MimeStream make_mem() { + MimeStream mstream{g_mime_stream_mem_new()}; + mstream.unref(); /* remove extra ref */ + return mstream; + } + + static MimeStream make_filtered(MimeStream& stream) { + MimeStream mstream{g_mime_stream_filter_new(stream.self())}; + mstream.unref(); /* remove extra refs */ + return mstream; + } + + static MimeStream make_from_stream(GMimeStream *strm) { + MimeStream mstream{strm}; + mstream.unref(); /* remove extra ref */ + return mstream; + } + +private: + MimeStream(GMimeStream *stream): Object(G_OBJECT(stream)) { + if (!GMIME_IS_STREAM(self())) + throw std::runtime_error("not a mime-stream"); + }; + + GMimeStream* self() const { + return reinterpret_cast<GMimeStream*>(object()); + } +}; + +template<typename S, typename T> +constexpr Option<std::string_view> to_string_view_opt(const S& seq, T t) { + auto&& it = seq_find_if(seq, [&](auto&& item){return item.first == t;}); + if (it == seq.cend()) + return Nothing; + else + return it->second; +} + + +/** + * Thin wrapper around a GMimeDataWrapper + * + */ +struct MimeDataWrapper: public Object { + MimeDataWrapper(GMimeDataWrapper *wrapper): Object(G_OBJECT(wrapper)) { + if (!GMIME_IS_DATA_WRAPPER(self())) + throw std::runtime_error("not a data-wrapper"); + }; + + Result<size_t> write_to_stream(MimeStream& stream) const { + if (auto&& res = g_mime_data_wrapper_write_to_stream( + self(), GMIME_STREAM(stream.object())) ; res < 0) + return Err(Error::Code::Message, "failed to write to stream"); + else + return Ok(static_cast<size_t>(res)); + } + +private: + GMimeDataWrapper* self() const { + return reinterpret_cast<GMimeDataWrapper*>(object()); + } +}; + + + +/** + * Thin wrapper around a GMimeCertifcate + * + */ +struct MimeCertificate: public Object { + MimeCertificate(GMimeCertificate *cert) : Object{G_OBJECT(cert)} { + if (!GMIME_IS_CERTIFICATE(self())) + throw std::runtime_error("not a certificate"); + } + + enum struct PubkeyAlgo { + Default = GMIME_PUBKEY_ALGO_DEFAULT, + Rsa = GMIME_PUBKEY_ALGO_RSA, + RsaE = GMIME_PUBKEY_ALGO_RSA_E, + RsaS = GMIME_PUBKEY_ALGO_RSA_S, + ElgE = GMIME_PUBKEY_ALGO_ELG_E, + Dsa = GMIME_PUBKEY_ALGO_DSA, + Ecc = GMIME_PUBKEY_ALGO_ECC, + Elg = GMIME_PUBKEY_ALGO_ELG, + EcDsa = GMIME_PUBKEY_ALGO_ECDSA, + EcDh = GMIME_PUBKEY_ALGO_ECDH, + EdDsa = GMIME_PUBKEY_ALGO_EDDSA, + }; + + enum struct DigestAlgo { + Default = GMIME_DIGEST_ALGO_DEFAULT, + Md5 = GMIME_DIGEST_ALGO_MD5, + Sha1 = GMIME_DIGEST_ALGO_SHA1, + RipEmd160 = GMIME_DIGEST_ALGO_RIPEMD160, + Md2 = GMIME_DIGEST_ALGO_MD2, + Tiger192 = GMIME_DIGEST_ALGO_TIGER192, + Haval5160 = GMIME_DIGEST_ALGO_HAVAL5160, + Sha256 = GMIME_DIGEST_ALGO_SHA256, + Sha384 = GMIME_DIGEST_ALGO_SHA384, + Sha512 = GMIME_DIGEST_ALGO_SHA512, + Sha224 = GMIME_DIGEST_ALGO_SHA224, + Md4 = GMIME_DIGEST_ALGO_MD4, + Crc32 = GMIME_DIGEST_ALGO_CRC32, + Crc32Rfc1510 = GMIME_DIGEST_ALGO_CRC32_RFC1510, + Crc32Rfc2440 = GMIME_DIGEST_ALGO_CRC32_RFC2440, + }; + + enum struct Trust { + Unknown = GMIME_TRUST_UNKNOWN, + Undefined = GMIME_TRUST_UNDEFINED, + Never = GMIME_TRUST_NEVER, + Marginal = GMIME_TRUST_MARGINAL, + TrustFull = GMIME_TRUST_FULL, + TrustUltimate = GMIME_TRUST_ULTIMATE, + }; + + enum struct Validity { + Unknown = GMIME_VALIDITY_UNKNOWN, + Undefined = GMIME_VALIDITY_UNDEFINED, + Never = GMIME_VALIDITY_NEVER, + Marginal = GMIME_VALIDITY_MARGINAL, + Full = GMIME_VALIDITY_FULL, + Ultimate = GMIME_VALIDITY_ULTIMATE, + }; + + PubkeyAlgo pubkey_algo() const { + return static_cast<PubkeyAlgo>( + g_mime_certificate_get_pubkey_algo(self())); + } + + DigestAlgo digest_algo() const { + return static_cast<DigestAlgo>( + g_mime_certificate_get_digest_algo(self())); + } + + Validity id_validity() const { + return static_cast<Validity>( + g_mime_certificate_get_id_validity(self())); + } + + Trust trust() const { + return static_cast<Trust>( + g_mime_certificate_get_trust(self())); + } + + Option<std::string> issuer_serial() const { + return to_string_opt(g_mime_certificate_get_issuer_serial(self())); + } + Option<std::string> issuer_name() const { + return to_string_opt(g_mime_certificate_get_issuer_name(self())); + } + + Option<std::string> fingerprint() const { + return to_string_opt(g_mime_certificate_get_fingerprint(self())); + } + + Option<std::string> key_id() const { + return to_string_opt(g_mime_certificate_get_key_id(self())); + } + + + Option<std::string> name() const { + return to_string_opt(g_mime_certificate_get_name(self())); + } + + Option<std::string> user_id() const { + return to_string_opt(g_mime_certificate_get_user_id(self())); + } + + Option<::time_t> created() const { + if (auto t = g_mime_certificate_get_created(self()); t >= 0) + return t; + else + return Nothing; + } + + Option<::time_t> expires() const { + if (auto t = g_mime_certificate_get_expires(self()); t >= 0) + return t; + else + return Nothing; + } + +private: + GMimeCertificate* self() const { + return reinterpret_cast<GMimeCertificate*>(object()); + } +}; + +constexpr std::array<std::pair<MimeCertificate::PubkeyAlgo, std::string_view>, 11> +AllPubkeyAlgos = {{ + { MimeCertificate::PubkeyAlgo::Default, "default"}, + { MimeCertificate::PubkeyAlgo::Rsa, "rsa"}, + { MimeCertificate::PubkeyAlgo::RsaE, "rsa-encryption-only"}, + { MimeCertificate::PubkeyAlgo::RsaS, "rsa-signing-only"}, + { MimeCertificate::PubkeyAlgo::ElgE, "el-gamal-encryption-only"}, + { MimeCertificate::PubkeyAlgo::Dsa, "dsa"}, + { MimeCertificate::PubkeyAlgo::Ecc, "elliptic curve"}, + { MimeCertificate::PubkeyAlgo::Elg, "el-gamal"}, + { MimeCertificate::PubkeyAlgo::EcDsa, "elliptic-curve+dsa"}, + { MimeCertificate::PubkeyAlgo::EcDh, "elliptic-curve+diffie-helman"}, + { MimeCertificate::PubkeyAlgo::EdDsa, "elliptic-curve+dsa-2"} + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeCertificate::PubkeyAlgo algo) { + return to_string_view_opt(AllPubkeyAlgos, algo); +} + +constexpr std::array<std::pair<MimeCertificate::DigestAlgo, std::string_view>, 15> +AllDigestAlgos = {{ + { MimeCertificate::DigestAlgo::Default, "default"}, + { MimeCertificate::DigestAlgo::Md5, "md5"}, + { MimeCertificate::DigestAlgo::Sha1, "sha1"}, + { MimeCertificate::DigestAlgo::RipEmd160, "ripemd-160"}, + { MimeCertificate::DigestAlgo::Md2, "md2"}, + { MimeCertificate::DigestAlgo::Tiger192, "tiger-192"}, + { MimeCertificate::DigestAlgo::Haval5160, "haval-5-160"}, + { MimeCertificate::DigestAlgo::Sha256, "sha-256"}, + { MimeCertificate::DigestAlgo::Sha384, "sha-384"}, + { MimeCertificate::DigestAlgo::Sha512, "sha-512"}, + { MimeCertificate::DigestAlgo::Sha224, "sha-224"}, + { MimeCertificate::DigestAlgo::Md4, "md4"}, + { MimeCertificate::DigestAlgo::Crc32, "crc32"}, + { MimeCertificate::DigestAlgo::Crc32Rfc1510, "crc32-rfc1510"}, + { MimeCertificate::DigestAlgo::Crc32Rfc2440, "crc32-rfc2440"}, + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeCertificate::DigestAlgo algo) { + return to_string_view_opt(AllDigestAlgos, algo); +} + +constexpr std::array<std::pair<MimeCertificate::Trust, std::string_view>, 6> +AllTrusts = {{ + { MimeCertificate::Trust::Unknown, "unknown" }, + { MimeCertificate::Trust::Undefined, "undefined" }, + { MimeCertificate::Trust::Never, "never" }, + { MimeCertificate::Trust::Marginal, "marginal" }, + { MimeCertificate::Trust::TrustFull, "trust-full" }, + { MimeCertificate::Trust::TrustUltimate,"trust-ultimate" }, + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeCertificate::Trust trust) { + return to_string_view_opt(AllTrusts, trust); +} + +constexpr std::array<std::pair<MimeCertificate::Validity, std::string_view>, 6> +AllValidities = {{ + { MimeCertificate::Validity::Unknown, "unknown" }, + { MimeCertificate::Validity::Undefined, "undefined" }, + { MimeCertificate::Validity::Never, "never" }, + { MimeCertificate::Validity::Marginal, "marginal" }, + { MimeCertificate::Validity::Full, "full" }, + { MimeCertificate::Validity::Ultimate, "ultimate" }, + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeCertificate::Validity val) { + return to_string_view_opt(AllValidities, val); +} + + + +/** + * Thin wrapper around a GMimeSignature + * + */ +struct MimeSignature: public Object { + MimeSignature(GMimeSignature *sig) : Object{G_OBJECT(sig)} { + if (!GMIME_IS_SIGNATURE(self())) + throw std::runtime_error("not a signature"); + } + + /** + * Signature status + * + */ + enum struct Status { + Valid = GMIME_SIGNATURE_STATUS_VALID, + Green = GMIME_SIGNATURE_STATUS_GREEN, + Red = GMIME_SIGNATURE_STATUS_RED, + KeyRevoked = GMIME_SIGNATURE_STATUS_KEY_REVOKED, + KeyExpired = GMIME_SIGNATURE_STATUS_KEY_EXPIRED, + SigExpired = GMIME_SIGNATURE_STATUS_SIG_EXPIRED, + KeyMissing = GMIME_SIGNATURE_STATUS_KEY_MISSING, + CrlMissing = GMIME_SIGNATURE_STATUS_CRL_MISSING, + CrlTooOld = GMIME_SIGNATURE_STATUS_CRL_TOO_OLD, + BadPolicy = GMIME_SIGNATURE_STATUS_BAD_POLICY, + SysError = GMIME_SIGNATURE_STATUS_SYS_ERROR, + TofuConflict = GMIME_SIGNATURE_STATUS_TOFU_CONFLICT + }; + + Status status() const { return static_cast<Status>( + g_mime_signature_get_status(self())); } + + ::time_t created() const { return g_mime_signature_get_created(self()); } + ::time_t expires() const { return g_mime_signature_get_expires(self()); } + + + const MimeCertificate certificate() const { + return MimeCertificate{g_mime_signature_get_certificate(self())}; + } + +private: + GMimeSignature* self() const { + return reinterpret_cast<GMimeSignature*>(object()); + } +}; + +constexpr std::array<std::pair<MimeSignature::Status, std::string_view>, 12> +AllMimeSignatureStatuses= {{ + { MimeSignature::Status::Valid, "valid" }, + { MimeSignature::Status::Green, "green" }, + { MimeSignature::Status::Red, "red" }, + { MimeSignature::Status::KeyRevoked, "key-revoked" }, + { MimeSignature::Status::KeyExpired, "key-expired" }, + { MimeSignature::Status::SigExpired, "sig-expired" }, + { MimeSignature::Status::KeyMissing, "key-missing" }, + { MimeSignature::Status::CrlMissing, "crl-missing" }, + { MimeSignature::Status::CrlTooOld, "crl-too-old" }, + { MimeSignature::Status::BadPolicy, "bad-policy" }, + { MimeSignature::Status::SysError, "sys-error" }, + { MimeSignature::Status::TofuConflict, "tofu-confict" }, + }}; +MU_ENABLE_BITOPS(MimeSignature::Status); + +static inline std::string to_string(MimeSignature::Status status) { + std::string str; + for (auto&& item: AllMimeSignatureStatuses) { + if (none_of(item.first & status)) + continue; + if (!str.empty()) + str += ", "; + str += item.second; + } + if (str.empty()) + str = "none"; + + return str; +} + + + + +/** +* Thin wrapper around a GMimeDecryptResult + * + */ +struct MimeDecryptResult: public Object { + MimeDecryptResult (GMimeDecryptResult *decres) : Object{G_OBJECT(decres)} { + if (!GMIME_IS_DECRYPT_RESULT(self())) + throw std::runtime_error("not a decrypt-result"); + } + + std::vector<MimeCertificate> recipients() const noexcept; + std::vector<MimeSignature> signatures() const noexcept; + + enum struct CipherAlgo { + Default = GMIME_CIPHER_ALGO_DEFAULT, + Idea = GMIME_CIPHER_ALGO_IDEA, + Des3 = GMIME_CIPHER_ALGO_3DES, + Cast5 = GMIME_CIPHER_ALGO_CAST5, + Blowfish = GMIME_CIPHER_ALGO_BLOWFISH, + Aes = GMIME_CIPHER_ALGO_AES, + Aes192 = GMIME_CIPHER_ALGO_AES192, + Aes256 = GMIME_CIPHER_ALGO_AES256, + TwoFish = GMIME_CIPHER_ALGO_TWOFISH, + Camellia128 = GMIME_CIPHER_ALGO_CAMELLIA128, + Camellia192 = GMIME_CIPHER_ALGO_CAMELLIA192, + Camellia256 = GMIME_CIPHER_ALGO_CAMELLIA256 + }; + + CipherAlgo cipher() const noexcept { + return static_cast<CipherAlgo>( + g_mime_decrypt_result_get_cipher(self())); + } + + using DigestAlgo = MimeCertificate::DigestAlgo; + DigestAlgo mdc() const noexcept { + return static_cast<DigestAlgo>( + g_mime_decrypt_result_get_mdc(self())); + } + + Option<std::string> session_key() const noexcept { + return to_string_opt(g_mime_decrypt_result_get_session_key(self())); + } + +private: + GMimeDecryptResult* self() const { + return reinterpret_cast<GMimeDecryptResult*>(object()); + } +}; + +constexpr std::array<std::pair<MimeDecryptResult::CipherAlgo, std::string_view>, 12> +AllCipherAlgos= {{ + {MimeDecryptResult::CipherAlgo::Default, "default"}, + {MimeDecryptResult::CipherAlgo::Idea, "idea"}, + {MimeDecryptResult::CipherAlgo::Des3, "3des"}, + {MimeDecryptResult::CipherAlgo::Cast5, "cast5"}, + {MimeDecryptResult::CipherAlgo::Blowfish, "blowfish"}, + {MimeDecryptResult::CipherAlgo::Aes, "aes"}, + {MimeDecryptResult::CipherAlgo::Aes192, "aes192"}, + {MimeDecryptResult::CipherAlgo::Aes256, "aes256"}, + {MimeDecryptResult::CipherAlgo::TwoFish, "twofish"}, + {MimeDecryptResult::CipherAlgo::Camellia128, "camellia128"}, + {MimeDecryptResult::CipherAlgo::Camellia192, "camellia192"}, + {MimeDecryptResult::CipherAlgo::Camellia256, "camellia256"}, + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeDecryptResult::CipherAlgo algo) { + return to_string_view_opt(AllCipherAlgos, algo); +} + + +/** + * Thin wrapper around a GMimeCryptoContext + * + */ +struct MimeCryptoContext : public Object { + + /** + * Make a new PGP crypto context. + * + * For 'test-mode', pass a test-path; in this mode GPG will be setup + * in an isolated mode so it does not affect normal usage. + * + * @param testpath (for unit-tests) pass a path to an existing dir to + * create a pgp setup. For normal use, leave empty. + * + * @return A MimeCryptoContext or an error + */ + static Result<MimeCryptoContext> + make_gpg(const std::string& testpath={}) try { + if (!testpath.empty()) { + if (auto&& res = setup_gpg_test(testpath); !res) + return Err(res.error()); + } + MimeCryptoContext ctx(g_mime_gpg_context_new()); + ctx.unref(); /* remove extra ref */ + return Ok(std::move(ctx)); + } catch (...) { + return Err(Error::Code::Crypto, "failed to create crypto context"); + } + + static Result<MimeCryptoContext> + make(const std::string& protocol) { + auto ctx = g_mime_crypto_context_new(protocol.c_str()); + if (!ctx) + return Err(Error::Code::Crypto, + "unsupported protocol {}", protocol); + MimeCryptoContext mctx{ctx}; + mctx.unref(); /* remove extra ref */ + return Ok(std::move(mctx)); + } + + Option<std::string> encryption_protocol() const noexcept { + return to_string_opt(g_mime_crypto_context_get_encryption_protocol(self())); + } + Option<std::string> signature_protocol() const noexcept { + return to_string_opt(g_mime_crypto_context_get_signature_protocol(self())); + } + Option<std::string> key_exchange_protocol() const noexcept { + return to_string_opt(g_mime_crypto_context_get_key_exchange_protocol(self())); + } + + /** + * Imports a stream of keys/certificates contained within stream into + * the key/certificate database controlled by @this. + * + * @param stream + * + * @return number of keys imported, or an error. + */ + Result<size_t> import_keys(MimeStream& stream); + + /** + * Prototype for a request-password function. + * + * @param ctx the MimeCryptoContext making the request + * @param user_id the user_id of the password being requested + * @param prompt a string containing some helpful context for the prompt + * @param reprompt true if this password request is a reprompt due to a + * previously bad password response + * @param response a stream for the application to write the password to + * (followed by a newline '\n' character) + * + * @return nothing (Ok) or an error, + */ + using PasswordRequestFunc = + std::function<Result<void>( + const MimeCryptoContext& ctx, + const std::string& user_id, + const std::string& prompt, + bool reprompt, + MimeStream& response)>; + /** + * Set a function to request a password. + * + * @param pw_func password function. + */ + void set_request_password(PasswordRequestFunc pw_func); + + +private: + MimeCryptoContext(GMimeCryptoContext *ctx): Object{G_OBJECT(ctx)} { + if (!GMIME_IS_CRYPTO_CONTEXT(self())) + throw std::runtime_error("not a crypto-context"); + } + + static Result<void> setup_gpg_test(const std::string& testpath); + + GMimeCryptoContext* self() const { + return reinterpret_cast<GMimeCryptoContext*>(object()); + } +}; + + +/** + * Thin wrapper around a GMimeObject + * + */ +class MimeObject: public Object { +public: + /** + * Construct a new MimeObject. Take a ref on the obj + * + * @param mime_part mime-part pointer + */ + MimeObject(const Object& obj): Object{obj} { + if (!GMIME_IS_OBJECT(self())) + throw std::runtime_error("not a mime-object"); + } + MimeObject(GMimeObject *mobj): Object{G_OBJECT(mobj)} { + if (mobj && !GMIME_IS_OBJECT(self())) + throw std::runtime_error("not a mime-object"); + } + + /** + * Get a header from the MimeObject + * + * @param header the header to retrieve + * + * @return header value (UTF-8) or Nothing + */ + Option<std::string> header(const std::string& header) const noexcept; + + + /** + * Get all headers as pairs of name, value + * + * @return all headers + */ + std::vector<std::pair<std::string, std::string>> headers() const noexcept; + + + /** + * Get the content type + * + * @return the content-type or Nothing + */ + Option<MimeContentType> content_type() const noexcept { + auto ct{g_mime_object_get_content_type(self())}; + if (!ct) + return Nothing; + else + return MimeContentType(ct); + } + + Option<std::string> mime_type() const noexcept { + if (auto ct = content_type(); !ct) + return Nothing; + else + return ct->mime_type(); + } + + /** + * Get the content-type parameter + * + * @param param name of parameter + * + * @return the value of the parameter, or Nothing + */ + Option<std::string> content_type_parameter(const std::string& param) const noexcept { + return Mu::to_string_opt( + g_mime_object_get_content_type_parameter(self(), param.c_str())); + } + + /** + * Write this MimeObject to some stream + * + * @param f_opts formatting options + * @param stream the stream + * + * @return the number or bytes written or an error + */ + Result<size_t> write_to_stream(const MimeFormatOptions& f_opts, + MimeStream& stream) const; + /** + * Write the object to a string. + * + * @return + */ + Option<std::string> to_string_opt() const noexcept; + + /** + * Write object to a file + * + * @param path path to file + * @param overwrite if true, overwrite existing file, if it bqexists + * + * @return size of the wrtten file, or an error. + */ + Result<size_t> to_file(const std::string& path, bool overwrite) const noexcept; + + /* + * subtypes. + */ + + /** + * Is this a MimePart? + * + * @return true or false + */ + bool is_part() const { return GMIME_IS_PART(self()); } + + /** + * Is this a MimeMultiPart? + * + * @return true or false + */ + bool is_multipart() const { return GMIME_IS_MULTIPART(self());} + + /** + * Is this a MimeMultiPart? + * + * @return true or false + */ + bool is_multipart_encrypted() const { + return GMIME_IS_MULTIPART_ENCRYPTED(self()); + } + + /** + * Is this a MimeMultiPart? + * + * @return true or false + */ + bool is_multipart_signed() const { + return GMIME_IS_MULTIPART_SIGNED(self()); + } + + /** + * Is this a MimeMessage? + * + * @return true or false + */ + bool is_message() const { return GMIME_IS_MESSAGE(self());} + + /** + * Is this a MimeMessagePart? + * + * @return true orf alse + */ + bool is_message_part() const { return GMIME_IS_MESSAGE_PART(self());} + + /** + * Is this a MimeApplicationpkcs7Mime? + * + * @return true orf alse + */ + bool is_mime_application_pkcs7_mime() const { + return GMIME_IS_APPLICATION_PKCS7_MIME(self()); + } + + /** + * Callback for for_each(). See GMimeObjectForEachFunc. + * + */ + using ForEachFunc = std::function<void(const MimeObject& parent, + const MimeObject& part)>; + +private: + GMimeObject* self() const { + return reinterpret_cast<GMimeObject*>(object()); + } +}; + + +/** + * Thin wrapper around a GMimeMessage + * + */ +class MimeMessage: public MimeObject { +public: + /** + * Construct a MimeMessage + * + * @param obj an Object of the right type + */ + MimeMessage(const Object& obj): MimeObject(obj) { + if (!is_message()) + throw std::runtime_error("not a mime-message"); + } + + /** + * Make a MimeMessage from a file + * + * @param path path to the file + * + * @return a MimeMessage or an error. + */ + static Result<MimeMessage> make_from_file (const std::string& path); + + /** + * Make a MimeMessage from a string + * + * @param path path to the file + * + * @return a MimeMessage or an error. + */ + static Result<MimeMessage> make_from_text (const std::string& text); + + /** + * Get the contacts of a given type, or None for _all_ + * + * @param ctype contact type + * + * @return contacts + */ + Contacts contacts(Contact::Type ctype) const noexcept; + + /** + * Gets the message-id if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> message_id() const noexcept { + return Mu::to_string_opt(g_mime_message_get_message_id(self())); + } + + /** + * Gets the message-id if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> subject() const noexcept { + return Mu::to_string_opt(g_mime_message_get_subject(self())); + } + + /** + * Gets the date if it exists, or nullopt otherwise. + * + * @return a time_t value (expressed as a 64-bit number) or nullopt + */ + Option<int64_t> date() const noexcept; + + + /** + * Get the references for this message (including in-reply-to), in the + * order of older..newer; the first one would the oldest parent, and + * in-reply-to would be the last one (if any). These are de-duplicated, + * and known-fake references removed (see implementation) + * + * @return references. + */ + std::vector<std::string> references() const noexcept; + + + /** + * Recursively apply func tol all parts of this message + * + * @param func a function + */ + void for_each(const ForEachFunc& func) const noexcept; + +private: + GMimeMessage* self() const { + return reinterpret_cast<GMimeMessage*>(object()); + } +}; + +/** + * Thin wrapper around a GMimePart. + * + */ +class MimePart: public MimeObject { +public: + /** + * Construct a MimePart + * + * @param obj an Object of the right type + */ + MimePart(const Object& obj): MimeObject(obj) { + if (!is_part()) + throw std::runtime_error("not a mime-part"); + } + + /** + * Determines whether or not the part is an attachment based on the + * value of the Content-Disposition header. + * + * @return true or false + */ + bool is_attachment() const noexcept { + return g_mime_part_is_attachment(self()); + } + + /** + * Gets the value of the Content-Description for this mime part + * if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> content_description() const noexcept { + return Mu::to_string_opt(g_mime_part_get_content_description(self())); + } + + /** + * Gets the value of the Content-Id for this mime part + * if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> content_id() const noexcept { + return Mu::to_string_opt(g_mime_part_get_content_id(self())); + } + + /** + * Gets the value of the Content-Md5 header for this mime part + * if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> content_md5() const noexcept { + return Mu::to_string_opt(g_mime_part_get_content_md5(self())); + + } + + /** + * Verify the content md5 for the specified mime part. Returns false if + * the mime part does not contain a Content-MD5. + * + * @return true or false + */ + bool verify_content_md5() const noexcept { + return g_mime_part_verify_content_md5(self()); + } + + /** + * Gets the value of the Content-Location for this mime part if it + * exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> content_location() const noexcept { + return Mu::to_string_opt(g_mime_part_get_content_location(self())); + } + + + MimeDataWrapper content() const noexcept { + return MimeDataWrapper{g_mime_part_get_content(self())}; + } + + /** + * Gets the filename for this mime part if it exists, or nullopt + * otherwise. + * + * @return string or nullopt + */ + Option<std::string> filename() const noexcept { + return Mu::to_string_opt(g_mime_part_get_filename(self())); + } + + /** + * Size of content, in bytes + * + * @return size + */ + size_t size() const noexcept; + + /** + * Get as UTF-8 string + * + * @return a string, or NULL. + */ + Option<std::string> to_string() const noexcept; + + /** + * Write part to a file + * + * @param path path to file + * @param overwrite if true, overwrite existing file, if it bqexists + * + * @return size of the wrtten file, or an error. + */ + Result<size_t> to_file(const std::string& path, bool overwrite) + const noexcept; + + /** + * Types of Content Encoding. + * + */ + enum struct ContentEncoding { + Default = GMIME_CONTENT_ENCODING_DEFAULT, + SevenBit = GMIME_CONTENT_ENCODING_7BIT, + EightBit = GMIME_CONTENT_ENCODING_8BIT, + Binary = GMIME_CONTENT_ENCODING_BINARY, + Base64 = GMIME_CONTENT_ENCODING_BASE64, + QuotedPrintable = GMIME_CONTENT_ENCODING_QUOTEDPRINTABLE, + UuEncode = GMIME_CONTENT_ENCODING_UUENCODE + }; + + /** + * Gets the content encoding of the mime part. + * + * @return the content encoding + */ + ContentEncoding content_encoding() const noexcept { + const auto enc{g_mime_part_get_content_encoding(self())}; + g_return_val_if_fail(enc <= GMIME_CONTENT_ENCODING_UUENCODE, + ContentEncoding::Default); + return static_cast<ContentEncoding>(enc); + } + + + /** + * Types of OpenPGP data + * + */ + enum struct OpenPGPData { + None = GMIME_OPENPGP_DATA_NONE, + Encrypted = GMIME_OPENPGP_DATA_ENCRYPTED, + Signed = GMIME_OPENPGP_DATA_SIGNED, + PublicKey = GMIME_OPENPGP_DATA_PUBLIC_KEY, + PrivateKey = GMIME_OPENPGP_DATA_PRIVATE_KEY, + }; + + /** + * Gets whether or not (and what type) of OpenPGP data is contained + * + * @return OpenGPGData + */ + OpenPGPData openpgp_data() const noexcept { + const auto data{g_mime_part_get_openpgp_data(self())}; + g_return_val_if_fail(data <= GMIME_OPENPGP_DATA_PRIVATE_KEY, + OpenPGPData::None); + return static_cast<OpenPGPData>(data); + } + +private: + GMimePart* self() const { + return reinterpret_cast<GMimePart*>(object()); + } +}; + + + +/** + * Thin wrapper around a GMimeMessagePart. + * + */ +class MimeMessagePart: public MimeObject { +public: + /** + * Construct a MimeMessagePart + * + * @param obj an Object of the right type + */ + MimeMessagePart(const Object& obj): MimeObject(obj) { + if (!is_message_part()) + throw std::runtime_error("not a mime-message-part"); + } + + /** + * Get the MimeMessage for this MimeMessagePart. + * + * @return the MimeMessage or Nothing + */ + Option<MimeMessage> get_message() const { + auto msg{g_mime_message_part_get_message(self())}; + if (msg) + return MimeMessage(Object(G_OBJECT(msg))); + else + return Nothing; + } +private: + GMimeMessagePart* self() const { + return reinterpret_cast<GMimeMessagePart*>(object()); + } + +}; + /** + * Thin wrapper around a GMimeApplicationPkcs7Mime + * + */ +class MimeApplicationPkcs7Mime: public MimePart { +public: + /** + * Construct a MimeApplicationPkcs7Mime + * + * @param obj an Object of the right type + */ + MimeApplicationPkcs7Mime(const Object& obj): MimePart(obj) { + if (!is_mime_application_pkcs7_mime()) + throw std::runtime_error("not a mime-application-pkcs7-mime"); + } + + enum struct SecureMimeType { + CompressedData = GMIME_SECURE_MIME_TYPE_COMPRESSED_DATA, + EnvelopedData = GMIME_SECURE_MIME_TYPE_ENVELOPED_DATA, + SignedData = GMIME_SECURE_MIME_TYPE_SIGNED_DATA, + CertsOnly = GMIME_SECURE_MIME_TYPE_CERTS_ONLY, + Unknown = GMIME_SECURE_MIME_TYPE_UNKNOWN + }; + + SecureMimeType smime_type() const { + return static_cast<SecureMimeType>( + g_mime_application_pkcs7_mime_get_smime_type(self())); + } + +private: + GMimeApplicationPkcs7Mime* self() const { + return reinterpret_cast<GMimeApplicationPkcs7Mime*>(object()); + } +}; + + +/** + * Thin wrapper around a GMimeMultiPart + * + */ +class MimeMultipart: public MimeObject { +public: + /** + * Construct a MimeMultipart + * + * @param obj an Object of the right type + */ + MimeMultipart(const Object& obj): MimeObject(obj) { + if (!is_multipart()) + throw std::runtime_error("not a mime-multipart"); + } + + Option<MimePart> signed_content_part() const { + return part(GMIME_MULTIPART_SIGNED_CONTENT); + } + + Option<MimePart> signed_signature_part() const { + return part(GMIME_MULTIPART_SIGNED_SIGNATURE); + } + + Option<MimePart> encrypted_version_part() const { + return part(GMIME_MULTIPART_ENCRYPTED_VERSION); + } + + Option<MimePart> encrypted_content_part() const { + return part(GMIME_MULTIPART_ENCRYPTED_CONTENT); + } + + /** + * Recursively apply func to all parts + * + * @param func a function + */ + void for_each(const ForEachFunc& func) const noexcept; + +private: + // Note: the part may not be available if the message was marked as + // _signed_ or _encrypted_ because it contained a forwarded signed or + // encrypted message. + Option<MimePart> part(int index) const { + if (auto&& p{g_mime_multipart_get_part(self() ,index)}; + !GMIME_IS_PART(p)) + return Nothing; + else + return Some(MimeObject{p}); + } + + GMimeMultipart* self() const { + return reinterpret_cast<GMimeMultipart*>(object()); + } +}; + + +/** + * Thin wrapper around a GMimeMultiPartEncrypted + * + */ +class MimeMultipartEncrypted: public MimeMultipart { +public: + /** + * Construct a MimeMultipartEncrypted + * + * @param obj an Object of the right type + */ + MimeMultipartEncrypted(const Object& obj): MimeMultipart(obj) { + if (!is_multipart_encrypted()) + throw std::runtime_error("not a mime-multipart-encrypted"); + } + + enum struct DecryptFlags { + None = GMIME_DECRYPT_NONE, + ExportSessionKey = GMIME_DECRYPT_EXPORT_SESSION_KEY, + NoVerify = GMIME_DECRYPT_NO_VERIFY, + EnableKeyserverLookups = GMIME_DECRYPT_ENABLE_KEYSERVER_LOOKUPS, + EnableOnlineCertificateChecks = GMIME_DECRYPT_ENABLE_ONLINE_CERTIFICATE_CHECKS + }; + + using Decrypted = std::pair<MimeObject, MimeDecryptResult>; + Result<Decrypted> decrypt(const MimeCryptoContext& ctx, + DecryptFlags flags=DecryptFlags::None, + const std::string& session_key = {}) const noexcept; + +private: + GMimeMultipartEncrypted* self() const { + return reinterpret_cast<GMimeMultipartEncrypted*>(object()); + } +}; + +MU_ENABLE_BITOPS(MimeMultipartEncrypted::DecryptFlags); + + +/** + * Thin wrapper around a GMimeMultiPartSigned + * + */ +class MimeMultipartSigned: public MimeMultipart { +public: + /** + * Construct a MimeMultipartSigned + * + * @param obj an Object of the right type + */ + MimeMultipartSigned(const Object& obj): MimeMultipart(obj) { + if (!is_multipart_signed()) + throw std::runtime_error("not a mime-multipart-signed"); + } + + enum struct VerifyFlags { + None = GMIME_VERIFY_NONE, + EnableKeyserverLookups = GMIME_VERIFY_ENABLE_KEYSERVER_LOOKUPS, + EnableOnlineCertificateChecks = GMIME_VERIFY_ENABLE_ONLINE_CERTIFICATE_CHECKS + }; + + // Result<std::vector<MimeSignature>> verify(VerifyFlags vflags=VerifyFlags::None) const noexcept; + + Result<std::vector<MimeSignature>> verify(const MimeCryptoContext& ctx, + VerifyFlags vflags=VerifyFlags::None) const noexcept; + +private: + GMimeMultipartSigned* self() const { + return reinterpret_cast<GMimeMultipartSigned*>(object()); + } +}; + + +MU_ENABLE_BITOPS(MimeMultipartSigned::VerifyFlags); + +} // namespace Mu + + +#endif /* MU_MIME_OBJECT_HH__ */ diff --git a/lib/message/mu-priority.cc b/lib/message/mu-priority.cc new file mode 100644 index 0000000..9b57cea --- /dev/null +++ b/lib/message/mu-priority.cc @@ -0,0 +1,76 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-priority.hh" + +using namespace Mu; + +std::string +Mu::to_string(Priority prio) +{ + return std::string{priority_name(prio)}; +} + +/* + * tests... also build as runtime-tests, so we can get coverage info + */ +#ifdef BUILD_TESTS +#include <glib.h> +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + +[[maybe_unused]] static void +test_priority_to_char() +{ + static_assert(to_char(Priority::Low) == 'l'); + static_assert(to_char(Priority::Normal) == 'n'); + static_assert(to_char(Priority::High) == 'h'); +} + +[[maybe_unused]] static void +test_priority_from_char() +{ + static_assert(priority_from_char('l') == Priority::Low); + static_assert(priority_from_char('n') == Priority::Normal); + static_assert(priority_from_char('h') == Priority::High); + static_assert(priority_from_char('x') == Priority::Normal); +} + +[[maybe_unused]] static void +test_priority_name() +{ + static_assert(priority_name(Priority::Low) == "low"); + static_assert(priority_name(Priority::Normal) == "normal"); + static_assert(priority_name(Priority::High) == "high"); +} + + +#ifdef BUILD_TESTS +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/message/priority/to-char", test_priority_to_char); + g_test_add_func("/message/priority/from-char", test_priority_from_char); + g_test_add_func("/message/priority/name", test_priority_name); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-priority.hh b/lib/message/mu-priority.hh new file mode 100644 index 0000000..a4bded3 --- /dev/null +++ b/lib/message/mu-priority.hh @@ -0,0 +1,154 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_PRIORITY_HH__ +#define MU_PRIORITY_HH__ + +#include <array> +#include <string> +#include <string_view> +#include "mu-fields.hh" + +namespace Mu { +/** + * Message priorities + * + */ + +/** + * The priority ids + * + */ +enum struct Priority : char { + Low = 'l', /**< Low priority */ + Normal = 'n', /**< Normal priority */ + High = 'h', /**< High priority */ +}; + +/** + * Sequence of all message priorities. + */ +static constexpr std::array<Priority, 3> AllMessagePriorities = { + Priority::Low, Priority::Normal, Priority::High}; + +/** + * Get the char for some priority + * + * @param id an id + * + * @return the char + */ +constexpr char +to_char(Priority prio) +{ + return static_cast<char>(prio); +} + +/** + * Get the priority for some character; unknown ones + * become Normal. + * + * @param c some character + */ +constexpr Priority +priority_from_char(char c) +{ + switch (c) { + case 'l': + return Priority::Low; + case 'h': + return Priority::High; + case 'n': + default: + return Priority::Normal; + } +} + +/** + * Get the priority from their (internal) name, i.e., low/normal/high + * or shortcut. + * + * @param pname + * + * @return the priority or none + */ +static inline Option<Priority> +priority_from_name(std::string_view pname) +{ + if (pname == "low" || pname == "l") + return Priority::Low; + else if (pname == "high" || pname == "h") + return Priority::High; + else if (pname == "normal" || pname == "n") + return Priority::Normal; + else + return Nothing; +} + + +/** + * Get the name for a given priority + * + * @return the name + */ +constexpr std::string_view +priority_name(Priority prio) +{ + switch (prio) { + case Priority::Low: + return "low"; + case Priority::High: + return "high"; + case Priority::Normal: + default: + return "normal"; + } +} + +/** + * Get the name for a given priority (backward compatibility) + * + * @return the name + */ +constexpr const char* +priority_name_c_str(Priority prio) +{ + switch (prio) { + case Priority::Low: + return "low"; + case Priority::High: + return "high"; + case Priority::Normal: + default: + return "normal"; + } +} + +/** + * Get a the message priority as a string + * + * @param prio priority + * + * @return a string + */ +std::string to_string(Priority prio); + +} // namespace Mu + +#endif /*MU_PRIORITY_HH_*/ diff --git a/lib/message/test-mu-message.cc b/lib/message/test-mu-message.cc new file mode 100644 index 0000000..1e0962e --- /dev/null +++ b/lib/message/test-mu-message.cc @@ -0,0 +1,1125 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "utils/mu-test-utils.hh" +#include "mu-message.hh" +#include "mu-mime-object.hh" +#include <glib.h> +#include <regex> + +using namespace Mu; + +/* + * test message 1 + */ + +static void +test_message_mailing_list() +{ + constexpr const char *test_message_1 = +R"(Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 32F276963F + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:34 +0300 (EEST) +Message-Id: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: anon@example.com +To: sqlite-dev@sqlite.org +Mime-Version: 1.0 (Apple Message framework v926) +Date: Mon, 4 Aug 2008 11:40:49 +0200 +X-Mailer: Apple Mail (2.926) +Subject: [sqlite-dev] VM optimization inside sqlite3VdbeExec +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Sender: sqlite-dev-bounces@sqlite.org +Content-Length: 639 + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; +)"; + auto message{Message::make_from_text( + test_message_1, + "/home/test/Maildir/inbox/cur/1649279256.107710_1.evergrey:2,S")}; + g_assert_true(!!message); + assert_equal(message->path(), + "/home/test/Maildir/inbox/cur/1649279256.107710_1.evergrey:2,S"); + g_assert_true(message->maildir().empty()); + + g_assert_true(message->bcc().empty()); + + g_assert_true(!message->body_html()); + assert_equal(message->body_text().value_or(""), +R"(Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; +)"); + g_assert_true(message->cc().empty()); + g_assert_cmpuint(message->date(), ==, 1217842849); + g_assert_true(message->flags() == (Flags::MailingList | Flags::Seen)); + + const auto from{message->from()}; + g_assert_cmpuint(from.size(),==,1); + assert_equal(from.at(0).name, ""); + assert_equal(from.at(0).email, "anon@example.com"); + + assert_equal(message->mailing_list(), "sqlite-dev.sqlite.org"); + assert_equal(message->message_id(), + "83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net"); + + g_assert_true(message->priority() == Priority::Low); + g_assert_cmpuint(message->size(),==,::strlen(test_message_1)); + + /* text-based message use time({}) as their changed-time */ + g_assert_cmpuint(::time({}) - message->changed(), >=, 0); + g_assert_cmpuint(::time({}) - message->changed(), <=, 2); + + g_assert_true(message->references().empty()); + + assert_equal(message->subject(), + "[sqlite-dev] VM optimization inside sqlite3VdbeExec"); + + const auto to{message->to()}; + g_assert_cmpuint(to.size(),==,1); + assert_equal(to.at(0).name, ""); + assert_equal(to.at(0).email, "sqlite-dev@sqlite.org"); + + assert_equal(message->header("X-Mailer").value_or(""), "Apple Mail (2.926)"); + + auto all_contacts{message->all_contacts()}; + g_assert_cmpuint(all_contacts.size(), ==, 4); + seq_sort(all_contacts, [](auto&& c1, auto&& c2){return c1.email < c2.email; }); + assert_equal(all_contacts[0].email, "anon@example.com"); + assert_equal(all_contacts[1].email, "sqlite-dev-bounces@sqlite.org"); + assert_equal(all_contacts[2].email, "sqlite-dev@sqlite.org"); + assert_equal(all_contacts[3].email, "sqlite-dev@sqlite.org"); +} + + +static void +test_message_attachments(void) +{ + constexpr const char* msg_text = +R"(Return-Path: <foo@example.com> +Received: from pop.gmail.com [256.85.129.309] + by evergrey with POP3 (fetchmail-6.4.29) + for <djcb@localhost> (single-drop); Thu, 24 Mar 2022 20:12:40 +0200 (EET) +Sender: "Foo, Example" <foo@example.com> +User-agent: mu4e 1.7.11; emacs 29.0.50 +From: "Foo Example" <foo@example.com> +To: bar@example.com +Subject: =?utf-8?B?w6R0dMOkY2htZcOxdHM=?= +Date: Thu, 24 Mar 2022 20:04:39 +0200 +Organization: ACME Inc. +Message-Id: <3144HPOJ0VC77.3H1XTAG2AMTLH@"@WILSONB.COM> +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain + +Hello, +--=-=-= +Content-Type: image/jpeg +Content-Disposition: attachment; filename=file-01.bin +Content-Transfer-Encoding: base64 + +AAECAw== +--=-=-= +Content-Type: audio/ogg +Content-Disposition: inline; filename=/tmp/file-02.bin +Content-Transfer-Encoding: base64 + +BAUGBw== +--=-=-= +Content-Type: message/rfc822 +Content-Disposition: attachment; + filename="message.eml" + +From: "Fnorb" <fnorb@example.com> +To: Bob <bob@example.com> +Subject: news for you +Date: Mon, 28 Mar 2022 22:53:26 +0300 + +Attached message! + +--=-=-= +Content-Type: text/plain + +World! +--=-=-=-- +)"; + + auto message{Message::make_from_text(msg_text)}; + g_assert_true(!!message); + g_assert_true(message->has_mime_message()); + g_assert_true(message->path().empty()); + + g_assert_true(message->bcc().empty()); + g_assert_true(!message->body_html()); + assert_equal(message->body_text().value_or(""), R"(Hello,World!)"); + + g_assert_true(message->cc().empty()); + g_assert_cmpuint(message->date(), ==, 1648145079); + /* no Flags::Unread since it's a message without path */ + g_assert_true(message->flags() == (Flags::HasAttachment)); + + const auto from{message->from()}; + g_assert_cmpuint(from.size(),==,1); + assert_equal(from.at(0).name, "Foo Example"); + assert_equal(from.at(0).email, "foo@example.com"); + + // problem case: https://github.com/djcb/mu/issues/2232o + assert_equal(message->message_id(), + "3144HPOJ0VC77.3H1XTAG2AMTLH@\"@WILSONB.COM"); + + g_assert_true(message->path().empty()); + g_assert_true(message->priority() == Priority::Normal); + g_assert_cmpuint(message->size(),==,::strlen(msg_text)); + + /* text-based message use time({}) as their changed-time */ + g_assert_cmpuint(::time({}) - message->changed(), >=, 0); + g_assert_cmpuint(::time({}) - message->changed(), <=, 2); + + assert_equal(message->subject(), "ättächmeñts"); + + const auto cache_path{message->cache_path()}; + g_assert_true(!!cache_path); + + g_assert_cmpuint(message->parts().size(),==,5); + { + auto&& part{message->parts().at(0)}; + g_assert_false(!!part.raw_filename()); + assert_equal(part.mime_type().value(), "text/plain"); + assert_equal(part.to_string().value(), "Hello,"); + } + { + auto&& part{message->parts().at(1)}; + assert_equal(part.raw_filename().value(), "file-01.bin"); + assert_equal(part.mime_type().value(), "image/jpeg"); + // file consists of 4 bytes 0...3 + g_assert_cmpuint(part.to_string()->at(0), ==, 0); + g_assert_cmpuint(part.to_string()->at(1), ==, 1); + g_assert_cmpuint(part.to_string()->at(2), ==, 2); + g_assert_cmpuint(part.to_string()->at(3), ==, 3); + } + { + auto&& part{message->parts().at(2)}; + assert_equal(part.raw_filename().value(), "/tmp/file-02.bin"); + assert_equal(part.cooked_filename().value(), "file-02.bin"); + assert_equal(part.mime_type().value(), "audio/ogg"); + // file consistso of 4 bytes 4..7 + assert_equal(part.to_string().value(), "\004\005\006\007"); + const auto fpath{*cache_path + part.cooked_filename().value()}; + const auto res = part.to_file(fpath, true); + + g_assert_cmpuint(*res,==,4); + g_assert_cmpuint(::access(fpath.c_str(), R_OK), ==, 0); + } + + { + auto&& part{message->parts().at(3)}; + g_assert_true(part.mime_type() == "message/rfc822"); + + const auto fname{*cache_path + "/msgpart"}; + g_assert_cmpuint(part.to_file(fname, true).value_or(123), ==, 139); + g_assert_true(::access(fname.c_str(), F_OK) == 0); + } + + { + auto&& part{message->parts().at(4)}; + g_assert_false(!!part.raw_filename()); + g_assert_true(!!part.mime_type()); + assert_equal(part.mime_type().value(), "text/plain"); + assert_equal(part.to_string().value(), "World!"); + } +} + + +/* + * some test keys. + */ + +constexpr std::string_view pub_key = +R"(-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEYlbaNhYJKwYBBAHaRw8BAQdAEgxZnlN3mIwqV89zchjFlEby8OgrbrkT+yRN +hQhc+A+0LU11IFRlc3QgKG11IHRlc3Rpbmcga2V5KSA8bXVAZGpjYnNvZnR3YXJl +Lm5sPoiUBBMWCgA8FiEE/HZRT+2bPjARz29Cw7FsU49t3vAFAmJW2jYCGwMFCwkI +BwIDIgIBBhUKCQgLAgQWAgMBAh4HAheAAAoJEMOxbFOPbd7wJ2kBAIGmUDWYEPtn +qYTwhZIdZtTa4KJ3UdtTqey9AnxJ9mzAAQDRJOoVppj5wW2xRhgYP+ysN2iBUYGE +MhahOcNgxodbCLg4BGJW2jYSCisGAQQBl1UBBQEBB0D4Sp+GTVre7Cx5a8D3SwLJ +/bRAVGDwqI7PL9B/cMmCTwMBCAeIeAQYFgoAIBYhBPx2UU/tmz4wEc9vQsOxbFOP +bd7wBQJiVto2AhsMAAoJEMOxbFOPbd7w1tYA+wdfYCcwOP0QoNZZz2Yk12YkDk2R +FsRrZZpb0GKC/a2VAP4qFceeSegcUCBTQaoeFE9vq9XiUVOO98QI8r9C8QwvBw== +=jM/g +-----END PGP PUBLIC KEY BLOCK----- +)"; + +constexpr std::string_view priv_key = // "test1234" +R"(-----BEGIN PGP PRIVATE KEY BLOCK----- + +lIYEYlbaNhYJKwYBBAHaRw8BAQdAEgxZnlN3mIwqV89zchjFlEby8OgrbrkT+yRN +hQhc+A/+BwMCz6T2uBpk6a7/rXyE7C1bRbGjP6YSFcyRFz8VRV3Xlm7z6rdbdKZr +8R15AtLvXA4DOK5GiZRB2VbIxi8B9CtZ9qQx6YbQPkAmRzISGAjECrQtTXUgVGVz +dCAobXUgdGVzdGluZyBrZXkpIDxtdUBkamNic29mdHdhcmUubmw+iJQEExYKADwW +IQT8dlFP7Zs+MBHPb0LDsWxTj23e8AUCYlbaNgIbAwULCQgHAgMiAgEGFQoJCAsC +BBYCAwECHgcCF4AACgkQw7FsU49t3vAnaQEAgaZQNZgQ+2ephPCFkh1m1NrgondR +21Op7L0CfEn2bMABANEk6hWmmPnBbbFGGBg/7Kw3aIFRgYQyFqE5w2DGh1sInIsE +YlbaNhIKKwYBBAGXVQEFAQEHQPhKn4ZNWt7sLHlrwPdLAsn9tEBUYPCojs8v0H9w +yYJPAwEIB/4HAwI9MZDWcsoiJ/9oV5DRiAedeo3Ta/1M+aKfeNV36Ch1VGLwQF3E +V77qIrJlsT8CwOZHWUksUBENvG3ak3vd84awHHaHoTmoFwtISfvQrFK0iHgEGBYK +ACAWIQT8dlFP7Zs+MBHPb0LDsWxTj23e8AUCYlbaNgIbDAAKCRDDsWxTj23e8NbW +APsHX2AnMDj9EKDWWc9mJNdmJA5NkRbEa2WaW9Bigv2tlQD+KhXHnknoHFAgU0Gq +HhRPb6vV4lFTjvfECPK/QvEMLwc= +=w1Nc +-----END PGP PRIVATE KEY BLOCK----- +)"; + + +static void +test_message_signed(void) +{ + constexpr const char *msgtext = +R"(Return-Path: <diggler@gmail.com> +From: Mu Test <mu@djcbsoftware.nl> +To: Mu Test <mu@djcbsoftware.nl> +Subject: boo +Date: Wed, 13 Apr 2022 17:19:08 +0300 +Message-ID: <878rs9ysin.fsf@djcbsoftware.nl> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; + micalg=pgp-sha512; protocol="application/pgp-signature" + +--=-=-= +Content-Type: text/plain + +Sapperdeflap + +--=-=-= +Content-Type: application/pgp-signature; name="signature.asc" + +-----BEGIN PGP SIGNATURE----- + +iIkEARYKADEWIQT8dlFP7Zs+MBHPb0LDsWxTj23e8AUCYlbcLhMcbXVAZGpjYnNv +ZnR3YXJlLm5sAAoJEMOxbFOPbd7waIkA/jK1oY7OL8vrDoubNYxamy8HHmwtvO01 +Q46aYjxe0As6AP90bcAZ3dcn5RcTJaM0UhZssguawZ+tnriD3+5DPkMMCg== +=e32+ +-----END PGP SIGNATURE----- +--=-=-=-- +)"; + TempDir tempdir; + auto ctx{MimeCryptoContext::make_gpg(tempdir.path())}; + g_assert_true(!!ctx); + + auto stream{MimeStream::make_mem()}; + stream.write(pub_key.data(), pub_key.size()); + stream.reset(); + + auto imported = ctx->import_keys(stream); + g_assert_cmpuint(*imported, ==, 1); + + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/1649279777.107710_1.mindcrime:2,RS")}; + g_assert_true(!!message); + + g_assert_true(message->bcc().empty()); + assert_equal(message->body_text().value_or(""), "Sapperdeflap\n"); + g_assert_true(message->flags() == (Flags::Signed|Flags::Seen|Flags::Replied)); + + size_t n{}; + for (auto&& part: message->parts()) { + if (!part.is_signed()) + continue; + + const auto& mobj{part.mime_object()}; + if (!mobj.is_multipart_signed()) + continue; + + const auto mpart{MimeMultipartSigned(mobj)}; + const auto sigs{mpart.verify(*ctx)}; + if (!sigs) + mu_warning("{}", sigs.error().what()); + + g_assert_true(!!sigs); + g_assert_cmpuint(sigs->size(), ==, 1); + ++n; + } + + g_assert_cmpuint(n, ==, 1); +} + + +static void +test_message_signed_encrypted(void) +{ + constexpr const char *msgtext = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl +Subject: encrypted and signed +Date: Wed, 13 Apr 2022 17:32:30 +0300 +Message-ID: <87lew9xddt.fsf@djcbsoftware.nl> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- + +hF4DeEerj6WhdZASAQdAKdZwmugAlQA8c06Q5iQw4rwSADgfEWBTWlI6tDw7hEAw +0qSSeeQbA802qjG5TesaDVbFoPp1gOESt67HkJBABj9niwZLnjbzVRXKFoPTYabu +1MBWAQkCEO6kS0N73XQeJ9+nDkUacRX6sSgVM0j+nRdCGcrCQ8MOfLd9KUUBxpXy +r/rIBMpZGOIpKJnoZ2x75VsQIp/ADHLe9zzXVe0tkahXJqvLo26w3gn4NSEIEDp6 +4T/zMZImqGrENaixNmRiRSAnwPkLt95qJGOIqYhuW3X6hMRZyU4zDNwkAvnK+2Fv +Wjd+EmiFzh5tvCmPOSj556YFMV7UpFWO9VznXX/T5+f4i+95Lsm9Uotv/SiNtNQG +DPU3wiL347SzmPFXckknjlzSzDL1XbdbHdmoJs0uNnbaZxRwhkuTYbLHdpBZrBgR +C0bdoCx44QVU8HaZ2x91h3GoM/0q5bqM/rvCauwbokiJgAUrznecNPY= +=Ado7 +-----END PGP MESSAGE----- +--=-=-=-- +)"; + TempDir tempdir; + auto ctx{MimeCryptoContext::make_gpg(tempdir.path())}; + g_assert_true(!!ctx); + + /// test1234 + // ctx->set_request_password([](const MimeCryptoContext& ctx, + // const std::string& user_id, + // const std::string& prompt, + // bool reprompt, + // MimeStream& response)->Result<void> { + // return Err(Error::Code::Internal, "boo"); + // //return Ok(); + // }); + + { + auto stream{MimeStream::make_mem()}; + stream.write(priv_key.data(), priv_key.size()); + stream.write(pub_key.data(), pub_key.size()); + stream.reset(); + + + g_assert_cmpint(ctx->import_keys(stream).value_or(-1),==,1); + } + + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/1649279888.107710_1.mindcrime:2,FS")}; + g_assert_true(!!message); + g_assert_true(message->flags() == (Flags::Encrypted|Flags::Seen|Flags::Flagged)); + + size_t n{}; + for (auto&& part: message->parts()) { + + if (!part.is_encrypted()) + continue; + + g_assert_false(!!part.content_description()); + g_assert_false(part.is_attachment()); + g_assert_cmpuint(part.size(),==,0); + + const auto& mobj{part.mime_object()}; + if (!mobj.is_multipart_encrypted()) + continue; + + /* FIXME: make this work without user having to + * type password */ + + // const auto mpart{MimeMultipartEncrypted(mobj)}; + // const auto decres = mpart.decrypt(*ctx); + // assert_valid_result(decres); + + ++n; + } + + g_assert_cmpuint(n, ==, 1); +} + + +static void +test_message_multipart_mixed_rfc822(void) +{ + constexpr const char *msgtext = +R"(Content-Type: multipart/mixed; + boundary="Multipart_Tue_Sep__2_15:42:35_2014-1" + +--Multipart_Tue_Sep__2_15:42:35_2014-1 +Content-Type: message/rfc822 +)"; + auto message{Message::make_from_text(msgtext)}; + g_assert_true(!!message); + //g_assert_true(message->sexp().empty()); +} + + +static void +test_message_detect_attachment(void) +{ + constexpr const char *msgtext = +R"(From: "DUCK, Donald" <donald@example.com> +Date: Tue, 3 May 2022 10:26:26 +0300 +Message-ID: <SADKLAJCLKDJLAS-xheQjE__+hS-3tff=pTYpMUyGiJwNGF_DA@mail.gmail.com> +Subject: =?Windows-1252?Q?Purkuty=F6urakka?= +To: Hello <moika@example.com> +Cc: =?iso-8859-1?q?M=FCller=2C?= Mickey <Mickey.Mueller@example.com> +Content-Type: multipart/mixed; boundary="000000000000e687ed05de166d71" + +--000000000000e687ed05de166d71 +Content-Type: multipart/alternative; boundary="000000000000e687eb05de166d6f" + +--000000000000e687eb05de166d6f +Content-Type: text/plain; charset="UTF-8" +Content-Transfer-Encoding: quoted-printable + +fyi + +---------- Forwarded message --------- +From: Fooish Bar <foobar@example.com> +Date: Tue, 3 May 2022 at 08:59 +Subject: Ty=C3=B6t +To: "DUCK, Donald" <donald@example.com> + +Moi, + +-- + +--000000000000e687eb05de166d6f +Content-Type: text/html; charset="UTF-8" +Content-Transfer-Encoding: quoted-printable + +abc + +--000000000000e687eb05de166d6f-- +--000000000000e687ed05de166d71 +Content-Type: application/pdf; + name="test1.pdf" +Content-Disposition: attachment; + filename="test2.pdf" +Content-Transfer-Encoding: base64 +Content-ID: <18088cfd4bc5517c6321> +X-Attachment-Id: 18088cfd4bc5517c6321 + +JVBERi0xLjcKJeLjz9MKNyAwIG9iago8PCAvVHlwZSAvUGFnZSAvUGFyZW50IDEgMCBSIC9MYXN0 +TW9kaWZpZWQgKEQ6MjAyMjA1MDMwODU3MzYrMDMnMDAnKSAvUmVzb3VyY2VzIDIgMCBSIC9NZWRp +cmVmCjM1NjE4CiUlRU9GCg== +--000000000000e687ed05de166d71-- +)"; + auto message{Message::make_from_text(msgtext)}; + g_assert_true(!!message); + + g_assert_true(message->path().empty()); + + /* https://groups.google.com/g/mu-discuss/c/kCtrlxMXBjo */ + g_assert_cmpuint(message->cc().size(),==, 1); + assert_equal(message->cc().at(0).email, "Mickey.Mueller@example.com"); + assert_equal(message->cc().at(0).name, "Müller, Mickey"); + assert_equal(message->cc().at(0).display_name(), "\"Müller, Mickey\" <Mickey.Mueller@example.com>"); + + g_assert_true(message->bcc().empty()); + assert_equal(message->subject(), "Purkutyöurakka"); + assert_equal(message->body_html().value_or(""), "abc\n"); + assert_equal(message->body_text().value_or(""), + R"(fyi + +---------- Forwarded message --------- +From: Fooish Bar <foobar@example.com> +Date: Tue, 3 May 2022 at 08:59 +Subject: Työt +To: "DUCK, Donald" <donald@example.com> + +Moi, + +-- +)"); + g_assert_cmpuint(message->date(), ==, 1651562786); + g_assert_true(message->flags() == (Flags::HasAttachment)); + + g_assert_cmpuint(message->parts().size(), ==, 3); + + for (auto&& part: message->parts()) + g_info("%s %s", + part.is_attachment() ? "yes" : "no", + part.mime_type().value_or("boo").c_str()); +} + + +static void +test_message_calendar(void) +{ + constexpr const char *msgtext = +R"(MIME-Version: 1.0 +From: William <william@example.com> +To: Billy <billy@example.com> +Date: Thu, 9 Jan 2014 11:09:34 +0100 +Subject: Invitation: HELLO, @ Thu 9 Jan 2014 08:30 - 09:30 + (william@example.com) +Thread-Topic: Invitation: HELLO, @ Thu 9 Jan 2014 08:30 - 09:30 + (william@example.com) +Thread-Index: Ac8NIuske7OtG01VRpukb/bHE7SVHg== +Message-ID: <001a11c3440066ee0b04ef86cea8@google.com> +Accept-Language: en-US +Content-Language: en-US +X-MS-Exchange-Organization-AuthAs: Anonymous +X-MS-Has-Attach: yes +Content-Type: multipart/mixed; + boundary="_004_001a11c3440066ee0b04ef86cea8googlecom_" + +--_004_001a11c3440066ee0b04ef86cea8googlecom_ +Content-Type: multipart/alternative; + boundary="_002_001a11c3440066ee0b04ef86cea8googlecom_" + +--_002_001a11c3440066ee0b04ef86cea8googlecom_ +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: base64 + +PGh0bWw+DQo8aGVhZD4NCjxtZXRhIGh0dHAtZXF1aXY9IkNvbnRlbnQtVHlwZSIgY29udGVudD0i +dGV4dC9odG1sOyBjaGFyc2V0PXV0Zi04Ij4NCjxtZXRhIG5hbWU9IkdlbmVyYXRvciIgY29udGVu +dD0iTWljcm9zb2Z0IEV4Y2hhbmdlIFNlcnZlciI+DQo8IS0tIGNvbnZlcnRlZCBmcm9tIHJ0ZiAt +LT4NCjxzdHlsZT48IS0tIC5FbWFpbFF1b3RlIHsgbWFyZ2luLWxlZnQ6IDFwdDsgcGFkZGluZy1s +ZWZ0OiA0cHQ7IGJvcmRlci1sZWZ0OiAjODAwMDAwIDJweCBzb2xpZDsgfSAtLT48L3N0eWxlPg0K +PC9oZWFkPg0KPGJvZHk+DQo8Zm9udCBmYWNlPSJUaW1lcyBOZXcgUm9tYW4iIHNpemU9IjMiPjxh +IG5hbWU9IkJNX0JFR0lOIj48L2E+DQo8dGFibGUgYm9yZGVyPSIxIiB3aWR0aD0iNzM0IiBzdHls +ZT0iYm9yZGVyOjEgc29saWQ7IGJvcmRlci1jb2xsYXBzZTpjb2xsYXBzZTsgbWFyZ2luLWxlZnQ6 +IDJwdDsgIj4NCjx0cj4NCjx0ZD48Zm9udCBzaXplPSIxIj48YSBocmVmPSJodHRwczovL3d3dy5n +b29nbGUuY29tL2NhbGVuZGFyL2V2ZW50P2FjdGlvbj1WSUVXJmFtcDtlaWQ9YzNOemNXUXhjRGxs +Ym1VeU0ySnZNbWsyYjNOeU56ZG5jRzhnWkdwallrQmthbU5pYzI5bWRIZGhjbVV1Ym13JmFtcDt0 +b2s9TWpZamQybHNiR2xoYlhOZlpESXdRR2RzYjJKdmJXRnBiQzVqYjIxak16YzJZVGhrWW1Ga016 +QTJaRFV3TldVMlltWXhOamRqTm1ZMVlUVXhObUpqTWpFNU4yWTMmYW1wO2N0ej1BbWVyaWNhL1Nh +b19QYXVsbyZhbXA7aGw9ZW5fR0IiPjxmb250IGNvbG9yPSIjMjIwMENDIj48dT5tb3JlDQpkZXRh +aWxzIMK7PC91PjwvZm9udD48L2E+PGJyPg0KDQo8ZGl2IHN0eWxlPSJtYXJnaW4tYm90dG9tOiAx +NHB0OyAiPjxmb250IGZhY2U9IkFyaWFsLCBzYW5zLXNlcmlmIiBzaXplPSIyIiBjb2xvcj0iIzIy +MjIyMiI+PGI+SEVMTE8sPC9iPjwvZm9udD48L2Rpdj4NCjxkaXY+PGZvbnQgc2l6ZT0iMSIgY29s +b3I9IiMyMjIyMjIiPjxicj4NCg0KSSBBTSBERVNNT05EIFdJTExJQU1TIEFORCBNWSBMSVRUTEUg +U0lTVEVSIElTIEdMT1JJQSwgT1VSIEZBVEhFUiBPV05TIEEgTElNSVRFRCBPRiBDT0NPQSBBTkQg +R09MRCBCVVNJTkVTUyBJTiBSRVBVQkxJUVVFIERVIENPTkdPLiBBRlRFUiBISVMgVFJJUCBUTyBD +T1RFIERJVk9JUkUgVE8gTkVHT1RJQVRFIE9OIENPQ09BIEFORCBHT0xEIEJVU0lORVNTIEhFIFdB +TlRFRCBUTyBJTlZFU1QgSU4gQUJST0FELiA8L2ZvbnQ+PC9kaXY+DQo8ZGl2IHN0eWxlPSJtYXJn +aW4tdG9wOiAxNHB0OyBtYXJnaW4tYm90dG9tOiAxNHB0OyAiPjxmb250IHNpemU9IjMiPk9ORSBX +RUVLIEhFIENBTUUgQkFDSyBGUk9NIEhJUyBUUklQIFRPIEFCSURKQU4gSEUgSEFEIEEgTU9UT1Ig +QUNDSURFTlQgV0lUSCBPVVIgTU9USEVSIFdISUNIIE9VUiBNT1RIRVIgRElFRCBJTlNUQU5UTFkg +QlVUIE9VUiBGQVRIRVIgRElFRCBBRlRFUiBGSVZFIERBWVMgSU4gQSBQUklWQVRFIEhPU1BJVEFM +IElOIE9VUiBDT1VOVFJZLg0KSVQgV0FTIExJS0UgT1VSIEZBVEhFUiBLTkVXIEhFIFdBUyBHT0lO +RyBUTyBESUUgTUFZIEhJUyBHRU5UTEUgU09VTCBSRVNUIElOIFBSRUZFQ1QgUEVBQ0UuIDwvZm9u +dD48L2Rpdj4NCjxkaXYgc3R5bGU9Im1hcmdpbi10b3A6IDE0cHQ7IG1hcmdpbi1ib3R0b206IDE0 +cHQ7ICI+PGZvbnQgc2l6ZT0iMyI+SEUgRElTQ0xPU0VEIFRPIE1FIEFTIFRIRSBPTkxZIFNPTiBU +SEFUIEhFIERFUE9TSVRFRCBUSEUgU1VNIE9GIChVU0QgJCAxMCw1MDAsMDAwKSBJTlRPIEEgQkFO +SyBJTiBBQklESkFOIFRIQVQgVEhFIE1PTkVZIFdBUyBNRUFOVCBGT1IgSElTIENPQ09BIEFORCBH +T0xEIEJVU0lORVNTIEhFIFdBTlRFRCBUTyBFU1RBQkxJU0ggSU4NCkFCUk9BRC5XRSBBUkUgU09M +SUNJVElORyBGT1IgWU9VUiBIRUxQIFRPIFRSQU5TRkVSIFRISVMgTU9ORVkgSU5UTyBZT1VSIEFD +Q09VTlQgSU4gWU9VUiBDT1VOVFJZIEZPUiBPVVIgSU5WRVNUTUVOVC4gPC9mb250PjwvZGl2Pg0K +PGRpdiBzdHlsZT0ibWFyZ2luLXRvcDogMTRwdDsgbWFyZ2luLWJvdHRvbTogMTRwdDsgIj48Zm9u +dCBzaXplPSIzIj5QTEVBU0UgRk9SIFNFQ1VSSVRZIFJFQVNPTlMsSSBBRFZJQ0UgWU9VIFJFUExZ +IFVTIFRIUk9VR0ggT1VSIFBSSVZBVEUgRU1BSUw6IDxhIGhyZWY9Im1haWx0bzp3aWxsaWFtc2Rl +c21vbmQxMDdAeWFob28uY29tLnZuIj48Zm9udCBjb2xvcj0iIzAwMDBGRiI+PHU+d2lsbGlhbXNk +ZXNtb25kMTA3QHlhaG9vLmNvbS52bjwvdT48L2ZvbnQ+PC9hPg0KRk9SIE1PUkUgREVUQUlMUy4g +PC9mb250PjwvZGl2Pg0KPGRpdiBzdHlsZT0ibWFyZ2luLXRvcDogMTRwdDsgbWFyZ2luLWJvdHRv +bTogMTRwdDsgIj48Zm9udCBzaXplPSIzIj5SRUdBUkRTLiA8L2ZvbnQ+PC9kaXY+DQo8ZGl2IHN0 +eWxlPSJtYXJnaW4tdG9wOiAxNHB0OyBtYXJnaW4tYm90dG9tOiAxNHB0OyAiPjxmb250IHNpemU9 +IjMiPkRFU01PTkQgL0dMT1JJQSBXSUxMSUFNUy48L2ZvbnQ+PC9kaXY+DQo8ZGl2Pjxmb250IHNp +emU9IjMiIGNvbG9yPSIjMjIyMjIyIj4mbmJzcDs8L2ZvbnQ+PC9kaXY+DQo8ZGl2Pjxmb250IHNp +emU9IjMiIGNvbG9yPSIjMjIyMjIyIj4mbmJzcDs8L2ZvbnQ+PC9kaXY+DQo8ZGl2Pjxmb250IHNp +emU9IjMiIGNvbG9yPSIjMjIyMjIyIj4mbmJzcDs8L2ZvbnQ+PC9kaXY+DQo8ZGl2Pjxmb250IHNp +emU9IjMiIGNvbG9yPSIjMjIyMjIyIj4mbmJzcDs8L2ZvbnQ+PC9kaXY+DQo8dGFibGUgYm9yZGVy +PSIxIiB3aWR0aD0iNzM0IiBzdHlsZT0iYm9yZGVyOjEgc29saWQ7IGJvcmRlci1jb2xsYXBzZTpj +b2xsYXBzZTsgbWFyZ2luLWxlZnQ6IDJwdDsgIj4NCjxjb2wgd2lkdGg9IjM2NSI+DQo8Y29sIHdp +ZHRoPSIzNjkiPg0KPHRyPg0KPHRkPjxmb250IHNpemU9IjMiPjxpPldoZW48L2k+PC9mb250Pjwv +dGQ+DQo8dGQ+PGZvbnQgZmFjZT0iQXJpYWwsIHNhbnMtc2VyaWYiIHNpemU9IjEiIGNvbG9yPSIj +MjIyMjIyIj5UaHUgOSBKYW4gMjAxNCAwODozMCDigJMgMDk6MzAgPGZvbnQgY29sb3I9IiM4ODg4 +ODgiPlNhbyBQYXVsbzwvZm9udD48L2ZvbnQ+PC90ZD4NCjwvdHI+DQo8dHI+DQo8dGQ+PGZvbnQg +c2l6ZT0iMyI+PGk+Q2FsZW5kYXI8L2k+PC9mb250PjwvdGQ+DQo8dGQ+PGZvbnQgZmFjZT0iQXJp +YWwsIHNhbnMtc2VyaWYiIHNpemU9IjEiIGNvbG9yPSIjMjIyMjIyIj53aWxsaWFtc19kMjBAZ2xv +Ym9tYWlsLmNvbTwvZm9udD48L3RkPg0KPC90cj4NCjx0cj4NCjx0ZD48Zm9udCBzaXplPSIzIj48 +aT5XaG88L2k+PC9mb250PjwvdGQ+DQo8dGQ+PGZvbnQgZmFjZT0iQXJpYWwsIHNhbnMtc2VyaWYi +IHNpemU9IjEiIGNvbG9yPSIjMjIyMjIyIj4oR3Vlc3QgbGlzdCBoYXMgYmVlbiBoaWRkZW4gYXQg +b3JnYW5pc2VyJ3MgcmVxdWVzdCk8L2ZvbnQ+PC90ZD4NCjwvdHI+DQo8L3RhYmxlPg0KPGRpdiBz +dHlsZT0ibWFyZ2luLWJvdHRvbTogMTRwdDsgIj48Zm9udCBzaXplPSIxIiBjb2xvcj0iIzg4ODg4 +OCI+R29pbmc/Jm5ic3A7Jm5ic3A7IDxhIGhyZWY9Imh0dHBzOi8vd3d3Lmdvb2dsZS5jb20vY2Fs +ZW5kYXIvZXZlbnQ/YWN0aW9uPVJFU1BPTkQmYW1wO2VpZD1jM056Y1dReGNEbGxibVV5TTJKdk1t +azJiM055TnpkbmNHOGdaR3BqWWtCa2FtTmljMjltZEhkaGNtVXVibXcmYW1wO3JzdD0xJmFtcDt0 +b2s9TWpZamQybHNiR2xoYlhOZlpESXdRR2RzYjJKdmJXRnBiQzVqYjIxak16YzJZVGhrWW1Ga016 +QTJaRFV3TldVMlltWXhOamRqTm1ZMVlUVXhObUpqTWpFNU4yWTMmYW1wO2N0ej1BbWVyaWNhL1Nh +b19QYXVsbyZhbXA7aGw9ZW5fR0IiPjxmb250IGNvbG9yPSIjMjIwMENDIj48dT48Yj5ZZXM8L2I+ +PC91PjwvZm9udD48L2E+PGZvbnQgY29sb3I9IiMyMjIyMjIiPjxiPg0KLSA8L2I+PC9mb250Pjxh +IGhyZWY9Imh0dHBzOi8vd3d3Lmdvb2dsZS5jb20vY2FsZW5kYXIvZXZlbnQ/YWN0aW9uPVJFU1BP +TkQmYW1wO2VpZD1jM056Y1dReGNEbGxibVV5TTJKdk1tazJiM055TnpkbmNHOGdaR3BqWWtCa2Ft +TmljMjltZEhkaGNtVXVibXcmYW1wO3JzdD0zJmFtcDt0b2s9TWpZamQybHNiR2xoYlhOZlpESXdR +R2RzYjJKdmJXRnBiQzVqYjIxak16YzJZVGhrWW1Ga016QTJaRFV3TldVMlltWXhOamRqTm1ZMVlU +VXhObUpqTWpFNU4yWTMmYW1wO2N0ej1BbWVyaWNhL1Nhb19QYXVsbyZhbXA7aGw9ZW5fR0IiPjxm +b250IGNvbG9yPSIjMjIwMENDIj48dT48Yj5NYXliZTwvYj48L3U+PC9mb250PjwvYT48Zm9udCBj +b2xvcj0iIzIyMjIyMiI+PGI+DQotIDwvYj48L2ZvbnQ+PGEgaHJlZj0iaHR0cHM6Ly93d3cuZ29v +Z2xlLmNvbS9jYWxlbmRhci9ldmVudD9hY3Rpb249UkVTUE9ORCZhbXA7ZWlkPWMzTnpjV1F4Y0Rs +bGJtVXlNMkp2TW1rMmIzTnlOemRuY0c4Z1pHcGpZa0JrYW1OaWMyOW1kSGRoY21VdWJtdyZhbXA7 +cnN0PTImYW1wO3Rvaz1NallqZDJsc2JHbGhiWE5mWkRJd1FHZHNiMkp2YldGcGJDNWpiMjFqTXpj +MllUaGtZbUZrTXpBMlpEVXdOV1UyWW1ZeE5qZGpObVkxWVRVeE5tSmpNakU1TjJZMyZhbXA7Y3R6 +PUFtZXJpY2EvU2FvX1BhdWxvJmFtcDtobD1lbl9HQiI+PGZvbnQgY29sb3I9IiMyMjAwQ0MiPjx1 +PjxiPk5vPC9iPjwvdT48L2ZvbnQ+PC9hPjxmb250IGNvbG9yPSIjMjIyMjIyIj4mbmJzcDsmbmJz +cDsmbmJzcDsNCjwvZm9udD48YSBocmVmPSJodHRwczovL3d3dy5nb29nbGUuY29tL2NhbGVuZGFy +L2V2ZW50P2FjdGlvbj1WSUVXJmFtcDtlaWQ9YzNOemNXUXhjRGxsYm1VeU0ySnZNbWsyYjNOeU56 +ZG5jRzhnWkdwallrQmthbU5pYzI5bWRIZGhjbVV1Ym13JmFtcDt0b2s9TWpZamQybHNiR2xoYlhO +ZlpESXdRR2RzYjJKdmJXRnBiQzVqYjIxak16YzJZVGhrWW1Ga016QTJaRFV3TldVMlltWXhOamRq +Tm1ZMVlUVXhObUpqTWpFNU4yWTMmYW1wO2N0ej1BbWVyaWNhL1Nhb19QYXVsbyZhbXA7aGw9ZW5f +R0IiPjxmb250IGNvbG9yPSIjMjIwMENDIj48dT5tb3JlDQpvcHRpb25zIMK7PC91PjwvZm9udD48 +L2E+PC9mb250PjwvZGl2Pg0KPC9mb250PjwvdGQ+DQo8L3RyPg0KPHRyPg0KPHRkIHN0eWxlPSJi +YWNrZ3JvdW5kLWNvbG9yOiAjRjZGNkY2OyAiPjxmb250IHNpemU9IjMiPkludml0YXRpb24gZnJv +bSA8YSBocmVmPSJodHRwczovL3d3dy5nb29nbGUuY29tL2NhbGVuZGFyLyI+PGZvbnQgY29sb3I9 +IiMwMDAwRkYiPjx1Pkdvb2dsZSBDYWxlbmRhcjwvdT48L2ZvbnQ+PC9hPg0KPGRpdiBzdHlsZT0i +bWFyZ2luLXRvcDogMTRwdDsgbWFyZ2luLWJvdHRvbTogMTRwdDsgIj48Zm9udCBzaXplPSIzIj5Z +b3UgYXJlIHJlY2VpdmluZyB0aGlzIGNvdXJ0ZXN5IGVtYWlsIGF0IHRoZSBhY2NvdW50IGRqY2JA +ZGpjYnNvZnR3YXJlLm5sIGJlY2F1c2UgeW91IGFyZSBhbiBhdHRlbmRlZSBvZiB0aGlzIGV2ZW50 +LjwvZm9udD48L2Rpdj4NCjxkaXYgc3R5bGU9Im1hcmdpbi10b3A6IDE0cHQ7IG1hcmdpbi1ib3R0 +b206IDE0cHQ7ICI+PGZvbnQgc2l6ZT0iMyI+VG8gc3RvcCByZWNlaXZpbmcgZnV0dXJlIG5vdGlm +aWNhdGlvbnMgZm9yIHRoaXMgZXZlbnQsIGRlY2xpbmUgdGhpcyBldmVudC4gQWx0ZXJuYXRpdmVs +eSwgeW91IGNhbiBzaWduIHVwIGZvciBhIEdvb2dsZSBhY2NvdW50IGF0DQo8YSBocmVmPSJodHRw +czovL3d3dy5nb29nbGUuY29tL2NhbGVuZGFyLyI+aHR0cHM6Ly93d3cuZ29vZ2xlLmNvbS9jYWxl +bmRhci88L2E+IGFuZCBjb250cm9sIHlvdXIgbm90aWZpY2F0aW9uIHNldHRpbmdzIGZvciB5b3Vy +IGVudGlyZSBjYWxlbmRhci48L2ZvbnQ+PC9kaXY+DQo8L2ZvbnQ+PC90ZD4NCjwvdHI+DQo8L3Rh +YmxlPg0KPC9mb250Pg0KPC9ib2R5Pg0KPC9odG1sPg0K + +--_002_001a11c3440066ee0b04ef86cea8googlecom_ +Content-Type: text/calendar; charset="UTF-8"; method=REQUEST +Content-Transfer-Encoding: 7bit + +BEGIN:VCALENDAR +PRODID:-//Google Inc//Google Calendar 70.9054//EN +VERSION:2.0 +CALSCALE:GREGORIAN +METHOD:REQUEST +BEGIN:VEVENT +DTSTART:20140109T103000Z +DTEND:20140109T113000Z +DTSTAMP:20140109T100934Z +ORGANIZER;CN=William:mailto:william@example.com +UID:sssqd1p9ene23bo2i6osr77gpo@google.com +ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-ACTION;RSVP= + TRUE;CN=billy@example.com;X-NUM-GUESTS=0:mailto:billy@example.com +CREATED:20140109T100932Z +DESCRIPTION:\nI AM DESMOND WILLIAMS AND MY LITTLE SISTER IS GLORIA\, OUR FA + THER OWNS A LIMITED OF COCOA AND GOLD BUSINESS IN REPUBLIQUE DU CONGO. AFTE + R HIS TRIP TO COTE DIVOIRE TO NEGOTIATE ON COCOA AND GOLD BUSINESS HE WANTE + D TO INVEST IN ABROAD. \n\nONE WEEK HE CAME BACK FROM HIS TRIP TO ABIDJAN H + E HAD A MOTOR ACCIDENT WITH OUR MOTHER WHICH OUR MOTHER DIED INSTANTLY BUT + OUR FATHER DIED AFTER FIVE DAYS IN A PRIVATE HOSPITAL IN OUR COUNTRY. IT WA + S LIKE OUR FATHER KNEW HE WAS GOING TO DIE MAY HIS GENTLE SOUL REST IN PREF + ECT PEACE. \n\nHE DISCLOSED TO ME AS THE ONLY SON THAT HE DEPOSITED THE SUM + OF (USD $ 10\,500\,000) INTO A BANK IN ABIDJAN THAT THE MONEY WAS MEANT FO + R HIS COCOA AND GOLD BUSINESS HE WANTED TO ESTABLISH IN ABROAD.WE ARE SOLIC + ITING FOR YOUR HELP TO TRANSFER THIS MONEY INTO YOUR ACCOUNT IN YOUR COUNTR + Y FOR OUR INVESTMENT. \n\nPLEASE FOR SECURITY REASONS\,I ADVICE YOU REPLY U + S THROUGH OUR PRIVATE EMAIL FOR MORE DETAI + LS. \n\nREGARDS. \n\nDESMOND /GLORIA WILLIAMS.\nView your event at http://w + ww.google.com/calendar/event?action=VIEW&eid=c3NzcWQxcDllbmUyM2JvMmk2b3NyNz + dncG8gZGpjYkBkamNic29mdHdhcmUubmw&tok=MjYjd2lsbGlhbXNfZDIwQGdsb2JvbWFpbC5jb + 21jMzc2YThkYmFkMzA2ZDUwNWU2YmYxNjdjNmY1YTUxNmJjMjE5N2Y3&ctz=America/Sao_Pau + lo&hl=en_GB. +LAST-MODIFIED:20140109T100932Z +LOCATION: +SEQUENCE:0 +STATUS:CONFIRMED +SUMMARY:HELLO\, +TRANSP:OPAQUE +END:VEVENT +END:VCALENDAR + +--_002_001a11c3440066ee0b04ef86cea8googlecom_-- + +--_004_001a11c3440066ee0b04ef86cea8googlecom_ +Content-Type: application/ics; name="invite.ics" +Content-Description: invite.ics +Content-Disposition: attachment; filename="invite.ics"; size=2029; + creation-date="Thu, 09 Jan 2014 10:09:44 GMT"; + modification-date="Thu, 09 Jan 2014 10:09:44 GMT" +Content-Transfer-Encoding: base64 + +QkVHSU46VkNBTEVOREFSDQpQUk9ESUQ6LS8vR29vZ2xlIEluYy8vR29vZ2xlIENhbGVuZGFyIDcw +LjkwNTQvL0VODQpWRVJTSU9OOjIuMA0KQ0FMU0NBTEU6R1JFR09SSUFODQpNRVRIT0Q6UkVRVUVT +VA0KQkVHSU46VkVWRU5UDQpEVFNUQVJUOjIwMTQwMTA5VDEwMzAwMFoNCkRURU5EOjIwMTQwMTA5 +VDExMzAwMFoNCkRUU1RBTVA6MjAxNDAxMDlUMTAwOTM0Wg0KT1JHQU5JWkVSO0NOPVdpbGxpYW1z +IFdpbGxpYW1zOm1haWx0bzp3aWxsaWFtc19kMjBAZ2xvYm9tYWlsLmNvbQ0KVUlEOnNzc3FkMXA5 +ZW5lMjNibzJpNm9zcjc3Z3BvQGdvb2dsZS5jb20NCkFUVEVOREVFO0NVVFlQRT1JTkRJVklEVUFM +O1JPTEU9UkVRLVBBUlRJQ0lQQU5UO1BBUlRTVEFUPU5FRURTLUFDVElPTjtSU1ZQPQ0KIFRSVUU7 +Q049ZGpjYkBkamNic29mdHdhcmUubmw7WC1OVU0tR1VFU1RTPTA6bWFpbHRvOmRqY2JAZGpjYnNv +ZnR3YXJlLm5sDQpDUkVBVEVEOjIwMTQwMTA5VDEwMDkzMloNCkRFU0NSSVBUSU9OOlxuSSBBTSBE +RVNNT05EIFdJTExJQU1TIEFORCBNWSBMSVRUTEUgU0lTVEVSIElTIEdMT1JJQVwsIE9VUiBGQQ0K +IFRIRVIgT1dOUyBBIExJTUlURUQgT0YgQ09DT0EgQU5EIEdPTEQgQlVTSU5FU1MgSU4gUkVQVUJM +SVFVRSBEVSBDT05HTy4gQUZURQ0KIFIgSElTIFRSSVAgVE8gQ09URSBESVZPSVJFIFRPIE5FR09U +SUFURSBPTiBDT0NPQSBBTkQgR09MRCBCVVNJTkVTUyBIRSBXQU5URQ0KIEQgVE8gSU5WRVNUIElO +IEFCUk9BRC4gXG5cbk9ORSBXRUVLIEhFIENBTUUgQkFDSyBGUk9NIEhJUyBUUklQIFRPIEFCSURK +QU4gSA0KIEUgSEFEIEEgTU9UT1IgQUNDSURFTlQgV0lUSCBPVVIgTU9USEVSIFdISUNIIE9VUiBN +T1RIRVIgRElFRCBJTlNUQU5UTFkgQlVUIA0KIE9VUiBGQVRIRVIgRElFRCBBRlRFUiBGSVZFIERB +WVMgSU4gQSBQUklWQVRFIEhPU1BJVEFMIElOIE9VUiBDT1VOVFJZLiBJVCBXQQ0KIFMgTElLRSBP +VVIgRkFUSEVSIEtORVcgSEUgV0FTIEdPSU5HIFRPIERJRSBNQVkgSElTIEdFTlRMRSBTT1VMIFJF +U1QgSU4gUFJFRg0KIEVDVCBQRUFDRS4gXG5cbkhFIERJU0NMT1NFRCBUTyBNRSBBUyBUSEUgT05M +WSBTT04gVEhBVCBIRSBERVBPU0lURUQgVEhFIFNVTQ0KICBPRiAoVVNEICQgMTBcLDUwMFwsMDAw +KSBJTlRPIEEgQkFOSyBJTiBBQklESkFOIFRIQVQgVEhFIE1PTkVZIFdBUyBNRUFOVCBGTw0KIFIg +SElTIENPQ09BIEFORCBHT0xEIEJVU0lORVNTIEhFIFdBTlRFRCBUTyBFU1RBQkxJU0ggSU4gQUJS +T0FELldFIEFSRSBTT0xJQw0KIElUSU5HIEZPUiBZT1VSIEhFTFAgVE8gVFJBTlNGRVIgVEhJUyBN +T05FWSBJTlRPIFlPVVIgQUNDT1VOVCBJTiBZT1VSIENPVU5UUg0KIFkgRk9SIE9VUiBJTlZFU1RN +RU5ULiBcblxuUExFQVNFIEZPUiBTRUNVUklUWSBSRUFTT05TXCxJIEFEVklDRSBZT1UgUkVQTFkg +VQ0KIFMgVEhST1VHSCBPVVIgUFJJVkFURSBFTUFJTDogd2lsbGlhbXNkZXNtb25kMTA3QHlhaG9v +LmNvbS52biBGT1IgTU9SRSBERVRBSQ0KIExTLiBcblxuUkVHQVJEUy4gXG5cbkRFU01PTkQgL0dM +T1JJQSBXSUxMSUFNUy5cblZpZXcgeW91ciBldmVudCBhdCBodHRwOi8vdw0KIHd3Lmdvb2dsZS5j +b20vY2FsZW5kYXIvZXZlbnQ/YWN0aW9uPVZJRVcmZWlkPWMzTnpjV1F4Y0RsbGJtVXlNMkp2TW1r +MmIzTnlOeg0KIGRuY0c4Z1pHcGpZa0JrYW1OaWMyOW1kSGRoY21VdWJtdyZ0b2s9TWpZamQybHNi +R2xoYlhOZlpESXdRR2RzYjJKdmJXRnBiQzVqYg0KIDIxak16YzJZVGhrWW1Ga016QTJaRFV3TldV +MlltWXhOamRqTm1ZMVlUVXhObUpqTWpFNU4yWTMmY3R6PUFtZXJpY2EvU2FvX1BhdQ0KIGxvJmhs +PWVuX0dCLg0KTEFTVC1NT0RJRklFRDoyMDE0MDEwOVQxMDA5MzJaDQpMT0NBVElPTjoNClNFUVVF +TkNFOjANClNUQVRVUzpDT05GSVJNRUQNClNVTU1BUlk6SEVMTE9cLA0KVFJBTlNQOk9QQVFVRQ0K +RU5EOlZFVkVOVA0KRU5EOlZDQUxFTkRBUg0K + +--_004_001a11c3440066ee0b04ef86cea8googlecom_-- + +)"; + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/162342449279256.107710_1.evergrey:2,PSp")}; + g_assert_true(!!message); + assert_equal(message->subject(), + "Invitation: HELLO, @ Thu 9 Jan 2014 08:30 - 09:30 (william@example.com)"); + g_assert_true(message->flags() == (Flags::Passed|Flags::Seen| + Flags::HasAttachment|Flags::Calendar)); + g_assert_cmpuint(message->body_html().value_or("").find("DETAILS"), ==, 2271); +} + + +static void +test_message_references() +{ + constexpr auto msgtext = +R"(Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=utf-8 +References: <YuvYh1JbE3v+abd5@kili> + <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> + <T4CDWjUrgtI5n4mh1JEdW6RLYzqbPE9-yDrhEVwDM22WX-198fBwcnLd-4_xR1gvsVSHQps9fp_pZevTF0ZmaA==@protonmail.internalid> +To: "Robin Murphy" <robin.murphy@arm.com> +Reply-To: "Dan Carpenter" <dan.carpenter@oracle.com> +From: "Dan Carpenter" <dan.carpenter@oracle.com> +Subject: Re: [PATCH] iommu/omap: fix buffer overflow in debugfs +List-Id: <kernel-janitors.vger.kernel.org> +Date: Fri, 5 Aug 2022 09:37:02 +0300 +In-Reply-To: <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> +Precedence: bulk +Message-Id: <20220805063702.GH3438@kadam> + +On Thu, Aug 04, 2022 at 05:31:39PM +0100, Robin Murphy wrote: +> On 04/08/2022 3:32 pm, Dan Carpenter wrote: +> > There are two issues here: +)"; + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/162342449279256.88888_1.evergrey:2,S")}; + g_assert_true(!!message); + assert_equal(message->subject(), + "Re: [PATCH] iommu/omap: fix buffer overflow in debugfs"); + g_assert_true(message->priority() == Priority::Low); + + /* + * "90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com" is seen both in + * references and in-reply-to; in the de-duplication, the first one wins. + */ + std::vector<std::string> expected_refs = { + "YuvYh1JbE3v+abd5@kili", + "90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com", + /* protonmail.internalid is fake and removed */ + // "T4CDWjUrgtI5n4mh1JEdW6RLYzqbPE9-yDrhEVwDM22WX-198fBwcnLd-4_" + // "xR1gvsVSHQps9fp_pZevTF0ZmaA==@protonmail.internalid" + }; + + assert_equal_seq_str(expected_refs, message->references()); +} + + +static void +test_message_outlook_body() +{ + constexpr auto msgtext = +R"x(Received: from vu-ex2.activedir.vu.lt (172.16.159.219) by + vu-ex1.activedir.vu.lt (172.16.159.218) with Microsoft SMTP Server + (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1118.9 + via Mailbox Transport; Fri, 27 May 2022 11:40:05 +0300 +Received: from vu-ex2.activedir.vu.lt (172.16.159.219) by + vu-ex2.activedir.vu.lt (172.16.159.219) with Microsoft SMTP Server + (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id + 15.2.1118.9; Fri, 27 May 2022 11:40:05 +0300 +Received: from vu-ex2.activedir.vu.lt ([172.16.159.219]) by + vu-ex2.activedir.vu.lt ([172.16.159.219]) with mapi id 15.02.1118.009; Fri, + 27 May 2022 11:40:05 +0300 +From: =?windows-1257?Q?XXXXXXXXXX= <XXXXXXXXXX> +To: <XXXXXXXXXX@XXXXXXXXXX.com> +Subject: =?windows-1257?Q?Pra=F0ymas?= +Thread-Topic: =?windows-1257?Q?Pra=F0ymas?= +Thread-Index: AQHYcaRi3ejPSLxkl0uTFDto7z2OcA== +Date: Fri, 27 May 2022 11:40:05 +0300 +Message-ID: <5c2cd378af634e929a6cc69da1e66b9d@XX.vu.lt> +Accept-Language: en-US, lt-LT +Content-Language: en-US +X-MS-Has-Attach: +Content-Type: text/html; charset="windows-1257" +Content-Transfer-Encoding: quoted-printable +MIME-Version: 1.0 +X-TUID: 1vFQ9RPwwg/u + +<html> +<head> +<meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Dwindows-1= +257"> +<style type=3D"text/css" style=3D"display:none;"><!-- P {margin-top:0;margi= +n-bottom:0;} --></style> +</head> +<body dir=3D"ltr"> +<div id=3D"divtagdefaultwrapper" style=3D"font-size:12pt;color:#000000;font= +-family:Calibri,Helvetica,sans-serif;" dir=3D"ltr"> +<p>Laba diena visiems,</p> +<p>Trumpai.</p> +<p>D=EBl leidimo ar neleidimo ginti darb=E0: ed=EBstytojo paskyroje spaud= +=FEiate ikon=E0 "ra=F0to darbai", atidar=E6 susiraskite =E1ra=F0= +=E0 "tvirtinti / netvirtinti", pa=FEym=EBkite vien=E0 i=F0 j=F8.&= +nbsp;</p> +<p><br> +</p> +<p>=D0=E1 darb=E0 privalu atlikti, kad paskui nekilt=F8 problem=F8 studentu= +i =E1vedant =E1vertinim=E0.</p> +<p><br> +</p> +<p>Jei neleid=FEiate ginti darbo, pra=F0au informuoti mane ir komisijos sek= +retori=F8.  </p> +<p><br> +</p> +<p>Vis=E0 tolesn=E6 informacij=E0 atsi=F8siu artimiausiu metu (stengsiuosi = +=F0iandien vakare).</p> +<p><br> +</p> +<p>Pagarbiai.</p> +<p><br> +</p> +<p><br> +</p> +<div id=3D"Signature"> +<div id=3D"divtagdefaultwrapper" dir=3D"ltr" style=3D"font-family: Calibri,= + Helvetica, sans-serif, EmojiFont, "Apple Color Emoji", "Seg= +oe UI Emoji", NotoColorEmoji, "Segoe UI Symbol", "Andro= +id Emoji", EmojiSymbols;"> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><br> +</p> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><br> +</p> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><br> +</p> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><span style=3D"font-size:10pt= +; background-color:rgb(255,255,255); color:rgb(0,111,201)"><br> +</span></p> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><span style=3D"font-size:10pt= +; background-color:rgb(255,255,255); color:rgb(0,111,201)">XXXXXXXXXX</span></p> +<p style=3D""><font color=3D"#006fc9"><span style=3D"font-size:13.3333px"><= +/span></font></p> +<span style=3D"font-size:10pt; background-color:rgb(255,255,255); color:rgb= +(0,111,201); font-size:10pt"></span> +<p style=3D""><font color=3D"#006fc9"><span style=3D"font-size:13.3333px">XXXXXXXXXX</span></font></p> +<p style=3D""><font color=3D"#006fc9"><span style=3D"font-size:13.3333px">XXXXXXXXXX</span></font></p> +<p style=3D""><font color=3D"#006fc9"><span style=3D"font-size:13.3333px">XXXXXXXXXX</span></font></p> +<p style=3D""><br> +</p> +<p style=3D""><br> +</p> +</div> +</div> +</div> +</body> +</html> +)x"; + g_test_bug("2349"); + + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/162342449279256.77777_1.evergrey:2,S")}; + g_assert_true(!!message); + + assert_equal(message->subject(), "PraÅ¡ymas"); + g_assert_true(message->priority() == Priority::Normal); + + g_assert_false(!!message->body_text()); + g_assert_true(!!message->body_html()); + g_assert_cmpuint(message->body_html()->find("<p>Pagarbiai.</p>"), ==, 935); +} + + +static void +test_message_message_id() +{ + constexpr const auto msg1 = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl +Message-ID: <87lew9xddt.fsf@djcbsoftware.nl> + +abc +)"; + + constexpr const auto msg2 = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl + +abc +)"; + + constexpr const auto msg3 = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl +Message-ID: + +abc +)"; + const auto m1{Message::make_from_text(msg1, "/foo/cur/m123:2,S")}; + assert_valid_result(m1); + + const auto m2{Message::make_from_text(msg2, "/foo/cur/m456:2,S")}; + assert_valid_result(m2); + const auto m3{Message::make_from_text(msg3, "/foo/cur/m789:2,S")}; + assert_valid_result(m3); + + assert_equal(m1->message_id(), "87lew9xddt.fsf@djcbsoftware.nl"); + + /* both with absent and empty message-id, generate "random" fake one, + * which must end in @mu.id */ + const auto id2{m2->message_id()}; + const auto id3{m3->message_id()}; + + g_assert_true(g_str_has_suffix(id2.c_str(), "@mu.id")); + g_assert_true(g_str_has_suffix(id3.c_str(), "@mu.id")); +} + + +static void +test_message_fail () +{ + { + const auto msg = Message::make_from_path("/root/non-existent-path-12345"); + g_assert_false(!!msg); + } + + { + const auto msg = Message::make_from_text("", ""); + g_assert_false(!!msg); + } +} + +static void +test_message_sanitize_maildir() +{ + assert_equal(Message::sanitize_maildir("/"), "/"); + assert_equal(Message::sanitize_maildir("/foo/bar"), "/foo/bar"); + assert_equal(Message::sanitize_maildir("/foo/bar/cuux/"), "/foo/bar/cuux"); +} + +static void +test_message_subject_with_newline() +{ +constexpr const auto txt = +R"(To: foo@example.com +Subject: =?utf-8?q?Le_poids_=C3=A9conomique_de_la_chasse_:_=0A=0Ala_dette_cach?= =?utf-8?q?=C3=A9e_de_la_chasse_!?= +Date: Mon, 24 Apr 2023 07:32:43 +0000 + +Hello! +)"; + g_test_bug("2477"); + + const auto msg{Message::make_from_text(txt, "/foo/cur/m123:2,S")}; + assert_valid_result(msg); + + assert_equal(msg->subject(), // newlines are filtered-out + "Le poids économique de la chasse : la dette cachée de la chasse !"); + assert_equal(msg->header("Subject").value_or(""), + "Le poids économique de la chasse : \n\nla dette cachée de la chasse !"); + g_assert_true(none_of(msg->flags() & Flags::MailingList)); +} + +static void +test_message_list_unsubscribe() +{ + constexpr const auto txt = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl +Subject: Test +Message-ID: <87lew9xddt.fsf@djcbsoftware.nl> +List-Unsubscribe: <mailto:unsubscribe-T7BC8RRQMK-booking-email-9@booking.com> + +abcdef +)"; + const auto msg{Message::make_from_text(txt, "/xxx/m123:2,S")}; + assert_valid_result(msg); + + assert_equal(msg->mailing_list(), ""); + g_assert_true(any_of(msg->flags() & Flags::MailingList)); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/message/message/mailing-list", + test_message_mailing_list); + g_test_add_func("/message/message/attachments", + test_message_attachments); + g_test_add_func("/message/message/signed", + test_message_signed); + g_test_add_func("/message/message/signed-encrypted", + test_message_signed_encrypted); + g_test_add_func("/message/message/multipart-mixed-rfc822", + test_message_multipart_mixed_rfc822); + g_test_add_func("/message/message/detect-attachment", + test_message_detect_attachment); + g_test_add_func("/message/message/calendar", + test_message_calendar); + g_test_add_func("/message/message/references", + test_message_references); + g_test_add_func("/message/message/outlook-body", + test_message_outlook_body); + g_test_add_func("/message/message/message-id", + test_message_message_id); + g_test_add_func("/message/message/subject-with-newline", + test_message_subject_with_newline); + g_test_add_func("/message/message/fail", + test_message_fail); + g_test_add_func("/message/message/sanitize-maildir", + test_message_sanitize_maildir); + g_test_add_func("/message/message/message-list-unsubscribe", + test_message_list_unsubscribe); + + return g_test_run(); +} diff --git a/lib/message/tests/meson.build b/lib/message/tests/meson.build new file mode 100644 index 0000000..94d0de9 --- /dev/null +++ b/lib/message/tests/meson.build @@ -0,0 +1,74 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# tests +# + +test('test-contact', + executable('test-contact', + '../mu-contact.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-document', + executable('test-document', + '../mu-document.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-fields', + executable('test-fields', + '../mu-fields.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-flags', + executable('test-flags', + '../mu-flags.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-message', + executable('test-message', + '../test-mu-message.cc', + install: false, + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-priority', + executable('test-priority', + '../mu-priority.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-message-file', + executable('test-message-file', + '../mu-message-file.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_message_dep])) + +test('test-message-part', + executable('test-message-part', + '../mu-message-part.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_message_dep])) diff --git a/lib/mu-config.cc b/lib/mu-config.cc new file mode 100644 index 0000000..6ce5c84 --- /dev/null +++ b/lib/mu-config.cc @@ -0,0 +1,126 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-config.hh" + +using namespace Mu; + +constexpr /*static*/ bool +validate_props() +{ + size_t id{0}; + for (auto&& prop: Config::properties) { + + // ids must match + if (static_cast<size_t>(prop.id) != id) + return false; + ++id; + } + + return true; +} + +#ifdef BUILD_TESTS +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + +[[maybe_unused]] +static void +test_props() +{ + static_assert(validate_props()); +} + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_basic() +{ + MemDb db; + Config conf_db{db}; + + g_assert_false(conf_db.read_only()); + + using Id = Config::Id; + + { + const auto rmd = conf_db.get<Id::RootMaildir>(); + g_assert_true(rmd.empty()); + } + + { + auto res = conf_db.set<Id::RootMaildir>("/home/djcb/Maildir"); + assert_valid_result(res); + + const auto rmd = conf_db.get<Id::RootMaildir>(); + assert_equal(rmd, "/home/djcb/Maildir"); + } + + { + g_assert_true(Config::property<Id::BatchSize>().default_val == "50000"); + g_assert_cmpuint(conf_db.get<Id::BatchSize>(),==,50000); + + assert_valid_result(conf_db.set<Id::BatchSize>(123456)); + g_assert_cmpuint(conf_db.get<Id::BatchSize>(),==,123456); + } + + + { + MemDb db2; + Config conf_db2{db2}; + + g_assert_cmpuint(conf_db2.get<Id::BatchSize>(),==,50000); + g_assert_true(conf_db2.get<Id::RootMaildir>().empty()); + + // BatchSize is configurable; RootMaildir is not. + conf_db2.import_configurable(conf_db); + + g_assert_cmpuint(conf_db2.get<Id::BatchSize>(),==,123456); + g_assert_true(conf_db2.get<Id::RootMaildir>().empty()); + } +} + +static void +test_read_only() +{ + MemDb db{true/*read-only*/}; + Config conf_db{db}; + + auto res = conf_db.set<Config::Id::MaxMessageSize>(12345); + g_assert_false(!!res); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/config-db/props", test_props); + g_test_add_func("/config-db/basic", test_basic); + g_test_add_func("/config-db/read-only", test_read_only); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-config.hh b/lib/mu-config.hh new file mode 100644 index 0000000..17924c7 --- /dev/null +++ b/lib/mu-config.hh @@ -0,0 +1,316 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_CONFIG_HH__ +#define MU_CONFIG_HH__ + +#include <cstdint> +#include <cinttypes> +#include <string_view> +#include <string> +#include <array> +#include <vector> +#include <variant> +#include <unordered_map> + +#include "mu-xapian-db.hh" + +#include <utils/mu-utils.hh> +#include <utils/mu-result.hh> +#include <utils/mu-option.hh> + +namespace Mu { + +struct Property { + enum struct Id { + BatchSize, /**< Xapian batch-size */ + Contacts, /**< Cache of contact information */ + Created, /**< Time of creation */ + IgnoredAddresses,/**< Email addresses ignored for the contacts-cache */ + LastChange, /**< Time of last change */ + LastIndex, /**< Time of last index */ + MaxMessageSize, /**< Maximum message size (in bytes) */ + PersonalAddresses, /**< List of personal e-mail addresses */ + RootMaildir, /**< Root maildir path */ + SchemaVersion, /**< Xapian DB schema version */ + SupportNgrams, /**< Support ngrams for indexing & querying + * for e.g. CJK languages */ + /* <private> */ + _count_ /* Number of Ids */ + }; + + static constexpr size_t id_size = static_cast<size_t>(Id::_count_); + /**< Number of Property::Ids */ + + enum struct Flags { + None = 0, /**< Nothing in particular */ + ReadOnly = 1 << 0, /**< Property is read-only for external use + * (but can change from within the store) */ + Configurable = 1 << 1, /**< A user-configurable parameter; name + * starts with 'conf-' */ + Internal = 1 << 2, /**< Mu-internal field */ + }; + enum struct Type { + Boolean, /**< Some boolean value */ + Number, /**< Some number */ + Timestamp, /**< Timestamp number */ + Path, /**< Path string */ + String, /**< A string */ + StringList, /**< A list of strings */ + }; + + using Value = std::variant<int64_t, std::string, std::vector<std::string> >; + + Id id; + Type type; + Flags flags; + std::string_view name; + std::string_view default_val; + std::string_view description; +}; + +MU_ENABLE_BITOPS(Property::Flags); + +class Config { +public: + using Id = Property::Id; + using Type = Property::Type; + using Flags = Property::Flags; + using Value = Property::Value; + + static constexpr std::array<Property, Property::id_size> + properties = {{ + { + Id::BatchSize, + Type::Number, + Flags::Configurable, + "batch-size", + "50000", + "Number of changes in a database transaction" + }, + { + Id::Contacts, + Type::String, + Flags::Internal, + "contacts", + {}, + "Serialized contact information" + }, + { + Id::Created, + Type::Timestamp, + Flags::ReadOnly, + MetadataIface::created_key, + {}, + "Database creation time" + }, + { + Id::IgnoredAddresses, + Type::StringList, + Flags::Configurable, + "ignored-addresses", + {}, + "E-mail addresses ignored for the contacts-cache, " + "literal or /regexp/" + }, + { + Id::LastChange, + Type::Timestamp, + Flags::ReadOnly, + MetadataIface::last_change_key, + {}, + "Time when last change occurred" + }, + { + Id::LastIndex, + Type::Timestamp, + Flags::ReadOnly, + "last-index", + {}, + "Time when last indexing operation was completed" + }, + { + Id::MaxMessageSize, + Type::Number, + Flags::Configurable, + "max-message-size", + "100000000", // default max: 100M bytes + "Maximum message size (in bytes); bigger messages are skipped" + }, + { + Id::PersonalAddresses, + Type::StringList, + Flags::Configurable, + "personal-addresses", + {}, + "Personal e-mail addresses, literal or /regexp/" + }, + { + Id::RootMaildir, + Type::Path, + Flags::ReadOnly, + "root-maildir", + {}, + "Absolute path of the top of the Maildir tree" + }, + { + Id::SchemaVersion, + Type::Number, + Flags::ReadOnly, + "schema-version", + {}, + "Version of the Xapian database schema" + }, + { + Id::SupportNgrams, + Type::Boolean, + Flags::Configurable, + "support-ngrams", + {}, + "Support n-grams for working with CJK and other languages" + }, + }}; + + /** + * Construct a new Config object. + * + * @param db The config-store (database); must stay valid for the + * lifetime of this config. + */ + Config(MetadataIface& cstore): cstore_{cstore}{} + + /** + * Get the property by its id + * + * @param id a property id (!= Id::_count_) + * + * @return the property + */ + template <Id ID> + constexpr static const Property& property() { + return properties[static_cast<size_t>(ID)]; + } + + /** + * Get a Property by its name. + * + * @param name The name + * + * @return the property or Nothing if not found + */ + static Option<const Property&> property(const std::string& name) { + const auto pname{std::string_view(name.data(), name.size())}; + for(auto&& prop: properties) + if (prop.name == pname) + return prop; + return Nothing; + } + + /** + * Get the property value of the correct type + * + * @param prop_id a property id + * + * @return the value or Nothing + */ + template<Id ID> + auto get() const { + constexpr auto&& prop{property<ID>()}; + const auto str = std::invoke([&]()->std::string { + const auto str = cstore_.metadata(std::string{prop.name}); + return str.empty() ? std::string{prop.default_val} : str; + }); + if constexpr (prop.type == Type::Number) + return static_cast<size_t>(str.empty() ? 0 : std::atoll(str.c_str())); + if constexpr (prop.type == Type::Boolean) + return static_cast<size_t>(str.empty() ? false : + std::atol(str.c_str()) != 0); + else if constexpr (prop.type == Type::Timestamp) + return static_cast<time_t>(str.empty() ? 0 : std::atoll(str.c_str())); + else if constexpr (prop.type == Type::Path || prop.type == Type::String) + return str; + else if constexpr (prop.type == Type::StringList) + return split(str, SepaChar1); + + throw std::logic_error("invalid prop " + std::string{prop.name}); + } + + /** + * Set a new value for some property + * + * @param prop_id property-id + * @param val the new value (of the correct type) + * + * @return Ok() or some error + */ + template<Id ID, typename T> + Result<void> set(const T& val) { + constexpr auto&& prop{property<ID>()}; + if (read_only()) + return Err(Error::Code::AccessDenied, + "cannot write to read-only db"); + + const auto strval = std::invoke([&]{ + if constexpr (prop.type == Type::Number || prop.type == Type::Timestamp) + return mu_format("{}", static_cast<int64_t>(val)); + if constexpr (prop.type == Type::Boolean) + return val ? "1" : "0"; + else if constexpr (prop.type == Type::Path || prop.type == Type::String) + return std::string{val}; + else if constexpr (prop.type == Type::StringList) + return join(val, SepaChar1); + else + throw std::logic_error("invalid prop " + std::string{prop.name}); + }); + + cstore_.set_metadata(std::string{prop.name}, strval); + return Ok(); + } + + /** + * Is this a read-only Config? + * + * + * @return true or false + */ + bool read_only() const { return cstore_.read_only();}; + + /** + * Import configurable settings to some other MetadataIface + * + * @param target some other metadata interface + */ + void import_configurable(const Config& src) const { + for (auto&& prop: properties) { + if (any_of(prop.flags & Flags::Configurable)) { + const auto&& key{std::string{prop.name}}; + if (auto&& val{src.cstore_.metadata(key)}; !val.empty()) + cstore_.set_metadata(key, std::string{val}); + } + } + } + +private: + MetadataIface& cstore_; +}; + + +} // namespace Mu + +#endif /* MU_CONFIG_DB_HH__ */ diff --git a/lib/mu-contacts-cache.cc b/lib/mu-contacts-cache.cc new file mode 100644 index 0000000..b9b9b50 --- /dev/null +++ b/lib/mu-contacts-cache.cc @@ -0,0 +1,609 @@ +/* +** Copyright (C) 2019-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-contacts-cache.hh" + +#include <mutex> +#include <unordered_map> +#include <set> +#include <sstream> +#include <functional> +#include <algorithm> +#include <ctime> + +#include <utils/mu-utils.hh> +#include <utils/mu-regex.hh> +#include <glib.h> + +using namespace Mu; + +struct EmailHash { + std::size_t operator()(const std::string& email) const { + return lowercase_hash(email); + } +}; +struct EmailEqual { + bool operator()(const std::string& email1, const std::string& email2) const { + return lowercase_hash(email1) == lowercase_hash(email2); + } +}; + +using ContactUMap = std::unordered_map<const std::string, Contact, EmailHash, EmailEqual>; +struct ContactsCache::Private { + Private(Config& config_db) + :config_db_{config_db}, + contacts_{deserialize(config_db_.get<Config::Id::Contacts>())}, + personal_plain_{make_matchers<Config::Id::PersonalAddresses>()}, + personal_rx_{make_rx_matchers<Config::Id::PersonalAddresses>()}, + ignored_plain_{make_matchers<Config::Id::IgnoredAddresses>()}, + ignored_rx_{make_rx_matchers<Config::Id::IgnoredAddresses>()}, + dirty_{0}, + email_rx_{unwrap(Regex::make(email_rx_str, G_REGEX_OPTIMIZE))} + {} + + ~Private() { serialize(); } + + ContactUMap deserialize(const std::string&) const; + void serialize() const; + + bool is_valid_email(const std::string& email) const { + return email_rx_.matches(email); + } + + Config& config_db_; + ContactUMap contacts_; + mutable std::mutex mtx_; + + const StringVec personal_plain_; + const std::vector<Regex> personal_rx_; + + const StringVec ignored_plain_; + const std::vector<Regex> ignored_rx_; + + mutable size_t dirty_; + Regex email_rx_; + +private: + static bool is_rx(const std::string& p) { + return p.size() >= 2 && p.at(0) == '/' && p.at(p.length() - 1) == '/'; + } + + template<Config::Id Id> StringVec make_matchers() const { + return seq_remove(config_db_.get<Id>(), is_rx); + } + template<Config::Id Id> std::vector<Regex> make_rx_matchers() const { + std::vector<Regex> rxvec; + for (auto&& p: config_db_.get<Id>()) { + + if (!is_rx(p)) + continue; + constexpr auto opts{static_cast<GRegexCompileFlags>(G_REGEX_OPTIMIZE|G_REGEX_CASELESS)}; + + const auto rxstr{p.substr(1, p.length() - 2)}; + try { + rxvec.push_back(unwrap(Regex::make(rxstr, opts))); + mu_debug("match {}: '{}' {}", Config::property<Id>().name, + p, rxvec.back()); + } catch (const Error& rex) { + mu_warning("invalid personal address regexp '{}': {}", + p, rex.what()); + } + } + return rxvec; + } + + /* regexp as per: + * https://html.spec.whatwg.org/multipage/input.html#valid-e-mail-address + * + * "This requirement is a willful violation of RFC 5322, which defines a + * syntax for email addresses that is simultaneously too strict (before + * the "@" character), too vague (after the "@" character), and too lax + * (allowing comments, whitespace characters, and quoted strings in + * manners unfamiliar to most users) to be of practical use here." + */ + static constexpr auto email_rx_str = + R"(^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$)"; + +}; + +ContactUMap +ContactsCache::Private::deserialize(const std::string& serialized) const +{ + ContactUMap contacts; + std::stringstream ss{serialized, std::ios_base::in}; + std::string line; + + while (getline(ss, line)) { + const auto parts = Mu::split(line, SepaChar2); + if (G_UNLIKELY(parts.size() != 5)) { + mu_warning("error: '{}'", line); + continue; + } + Contact ci(parts[0], // email + std::move(parts[1]), // name + (time_t)g_ascii_strtoll(parts[3].c_str(), NULL, 10), // message_date + parts[2][0] == '1' ? true : false, // personal + (std::size_t)g_ascii_strtoll(parts[4].c_str(), NULL, 10), // frequency + g_get_monotonic_time()); // tstamp + contacts.emplace(std::move(parts[0]), std::move(ci)); + } + + return contacts; +} + + +void +ContactsCache::Private::serialize() const +{ + if (config_db_.read_only()) { + if (dirty_ > 0) + mu_critical("dirty data in read-only ccache!"); // bug + return; + } + + std::string s; + std::unique_lock lock(mtx_); + + if (dirty_ == 0) + return; // nothing to do. + + for (auto& item : contacts_) { + const auto& ci{item.second}; + s += mu_format("{}{}{}{}{}{}{}{}{}\n", + ci.email, SepaChar2, + ci.name, SepaChar2, + ci.personal ? 1 : 0, SepaChar2, + ci.message_date, SepaChar2, + ci.frequency); + } + config_db_.set<Config::Id::Contacts>(s); + dirty_ = 0; +} + +ContactsCache::ContactsCache(Config& config_db) + : priv_{std::make_unique<Private>(config_db)} +{} + +ContactsCache::~ContactsCache() = default; + +void +ContactsCache::serialize() const +{ + if (priv_->config_db_.read_only()) + throw std::runtime_error("cannot serialize read-only contacts-cache"); + + priv_->serialize(); +} + +void +ContactsCache::add(Contact&& contact) +{ + /* we do _not_ cache invalid email addresses, so we won't offer them in completions etc. It + * should be _rare_, but we've seen cases ( broken local messages) */ + if (!is_valid(contact.email)) { + mu_warning("not caching invalid e-mail address '{}'", contact.email); + return; + } + + if (is_ignored(contact.email)) { + /* ignored this address, e.g. 'noreply@example.com */ + return; + } + + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + ++priv_->dirty_; + + auto it = priv_->contacts_.find(contact.email); + + if (it == priv_->contacts_.end()) { // completely new contact + + contact.name = contact.name; + if (!contact.personal) + contact.personal = is_personal(contact.email); + contact.tstamp = g_get_monotonic_time(); + + auto email{contact.email}; + // return priv_->contacts_.emplace(ContactUMap::value_type(email, std::move(contact))) + // .first->second; + mu_debug("adding contact {} <{}>", contact.name.c_str(), contact.email.c_str()); + priv_->contacts_.emplace(ContactUMap::value_type(email, std::move(contact))); + + } else { // existing contact. + auto& existing{it->second}; + ++existing.frequency; + if (contact.message_date > existing.message_date) { // update? + existing.email = std::move(contact.email); + // update name only if new one is not empty. + if (!contact.name.empty()) + existing.name = std::move(contact.name); + existing.tstamp = g_get_monotonic_time(); + existing.message_date = contact.message_date; + } + mu_debug("updating contact {} <{}> ({})", + contact.name, contact.email, existing.frequency); + } +} + + +void +ContactsCache::add(Contacts&& contacts, bool& personal) +{ + personal = seq_find_if(contacts,[&](auto&& c){ + return is_personal(c.email); }) != contacts.cend(); + + for (auto&& contact: contacts) { + contact.personal = personal; + add(std::move(contact)); + } +} + +const Contact* +ContactsCache::_find(const std::string& email) const +{ + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + const auto it = priv_->contacts_.find(email); + if (it == priv_->contacts_.end()) + return {}; + else + return &it->second; +} + +void +ContactsCache::clear() +{ + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + ++priv_->dirty_; + + priv_->contacts_.clear(); +} + +std::size_t +ContactsCache::size() const +{ + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + return priv_->contacts_.size(); +} + + +/** + * This is used for sorting the Contacts in order of relevance. A highly + * specific algorithm, but the details don't matter _too_ much. + * + * This is currently used for the ordering in mu-cfind and auto-completion in + * mu4e, if the various completion methods don't override it... + */ +constexpr auto RecentOffset{15 * 24 * 3600}; +struct ContactLessThan { + ContactLessThan() + : recently_{::time({}) - RecentOffset} {} + + bool operator()(const Mu::Contact& ci1, const Mu::Contact& ci2) const { + // non-personal is less relevant. + if (ci1.personal != ci2.personal) + return ci1.personal < ci2.personal; + + // older is less relevant for recent messages + if (std::max(ci1.message_date, ci2.message_date) > recently_ && + ci1.message_date != ci2.message_date) + return ci1.message_date < ci2.message_date; + + // less frequent is less relevant + if (ci1.frequency != ci2.frequency) + return ci1.frequency < ci2.frequency; + + // if all else fails, alphabetically + return ci1.email < ci2.email; + } + // only sort recently seen contacts by recency; approx 15 days. + // this changes during the lifetime, but that's all fine. + const time_t recently_; +}; + +using ContactSet = std::set<std::reference_wrapper<const Contact>, + ContactLessThan>; + +void +ContactsCache::for_each(const EachContactFunc& each_contact) const +{ + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + // first sort them for 'rank' + ContactSet sorted; + for (const auto& item : priv_->contacts_) + sorted.emplace(item.second); + + // return in _reverse_ order, so we get the most relevant ones first. + for (auto it = sorted.rbegin(); it != sorted.rend(); ++it) { + if (!each_contact(*it)) + break; + } +} + +static bool +address_matches(const std::string& addr, const StringVec& plain, const std::vector<Regex>& regexes) +{ + for (auto&& p : plain) + if (g_ascii_strcasecmp(addr.c_str(), p.c_str()) == 0) + return true; + + for (auto&& rx : regexes) { + if (rx.matches(addr)) + return true; + } + + return false; +} + +bool +ContactsCache::is_personal(const std::string& addr) const +{ + return address_matches(addr, priv_->personal_plain_, priv_->personal_rx_); +} + +bool +ContactsCache::is_ignored(const std::string& addr) const +{ + return address_matches(addr, priv_->ignored_plain_, priv_->ignored_rx_); +} + +bool +ContactsCache::is_valid(const std::string& addr) const +{ + return priv_->is_valid_email(addr); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_mu_contacts_cache_base() +{ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache contacts(cdb); + + g_assert_true(contacts.empty()); + g_assert_cmpuint(contacts.size(), ==, 0); + + contacts.add(Mu::Contact("foo.bar@example.com", + "Foo", {}, 12345)); + g_assert_false(contacts.empty()); + g_assert_cmpuint(contacts.size(), ==, 1); + + contacts.add(Mu::Contact("cuux@example.com", "Cuux", {}, + 54321)); + + g_assert_cmpuint(contacts.size(), ==, 2); + + contacts.add( + Mu::Contact("foo.bar@example.com", "Foo", {}, 77777)); + g_assert_cmpuint(contacts.size(), ==, 2); + + contacts.add( + Mu::Contact("Foo.Bar@Example.Com", "Foo", {}, 88888)); + g_assert_cmpuint(contacts.size(), ==, 2); + // note: replaces first. + + { + const auto info = contacts._find("bla@example.com"); + g_assert_false(info); + } + + { + const auto info = contacts._find("foo.BAR@example.com"); + g_assert_true(info); + + g_assert_cmpstr(info->email.c_str(), ==, "Foo.Bar@Example.Com"); + } + + contacts.clear(); + g_assert_true(contacts.empty()); + g_assert_cmpuint(contacts.size(), ==, 0); +} + +static void +test_mu_contacts_cache_personal() +{ + MemDb xdb{}; + Config cdb{xdb}; + cdb.set<Config::Id::PersonalAddresses> + (StringVec{{"foo@example.com", "bar@cuux.org", "/bar-.*@fnorb.f./"}}); + ContactsCache contacts{cdb}; + + g_assert_true(contacts.is_personal("foo@example.com")); + g_assert_true(contacts.is_personal("Bar@CuuX.orG")); + g_assert_true(contacts.is_personal("bar-123abc@fnorb.fi")); + g_assert_true(contacts.is_personal("bar-zzz@fnorb.fr")); + + g_assert_false(contacts.is_personal("foo@bar.com")); + g_assert_false(contacts.is_personal("BÂr@CuuX.orG")); + g_assert_false(contacts.is_personal("bar@fnorb.fi")); + g_assert_false(contacts.is_personal("bar-zzz@fnorb.xr")); +} + +static void +test_mu_contacts_cache_ignored() +{ + MemDb xdb{}; + Config cdb{xdb}; + cdb.set<Config::Id::IgnoredAddresses> + (StringVec{{"foo@example.com", "bar@cuux.org", "/bar-.*@fnorb.f./"}}); + ContactsCache contacts{cdb}; + + g_assert_true(contacts.is_ignored("foo@example.com")); + g_assert_true(contacts.is_ignored("Bar@CuuX.orG")); + g_assert_true(contacts.is_ignored("bar-123abc@fnorb.fi")); + g_assert_true(contacts.is_ignored("bar-zzz@fnorb.fr")); + + g_assert_false(contacts.is_ignored("foo@bar.com")); + g_assert_false(contacts.is_ignored("BÂr@CuuX.orG")); + g_assert_false(contacts.is_ignored("bar@fnorb.fi")); + g_assert_false(contacts.is_ignored("bar-zzz@fnorb.xr")); + + g_assert_cmpuint(contacts.size(),==,0); + contacts.add(Mu::Contact{"a@example.com", "a", 123, true, 1000, 0}); + g_assert_cmpuint(contacts.size(),==,1); + contacts.add(Mu::Contact{"foo@example.com", "b", 123, true, 1000, 0}); // ignored + contacts.add(Mu::Contact{"bar-123abc@fnorb.fi", "c", 123, true, 1000, 0}); // ignored + g_assert_cmpuint(contacts.size(),==,1); + contacts.add(Mu::Contact{"b@example.com", "d", 123, true, 1000, 0}); + g_assert_cmpuint(contacts.size(),==,2); +} + + + +static void +test_mu_contacts_cache_foreach() +{ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", 123, true, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", 456, true, 1000, 0}); + + { + size_t n{}; + g_assert_false(ccache.empty()); + g_assert_cmpuint(ccache.size(),==,2); + ccache.for_each([&](auto&& contact) { ++n; return false; }); + g_assert_cmpuint(n,==,1); + } + + { + size_t n{}; + g_assert_false(ccache.empty()); + g_assert_cmpuint(ccache.size(),==,2); + ccache.for_each([&](auto&& contact) { ++n; return true; }); + g_assert_cmpuint(n,==,2); + } + + { + size_t n{}; + ccache.clear(); + g_assert_true(ccache.empty()); + g_assert_cmpuint(ccache.size(),==,0); + ccache.for_each([&](auto&& contact) { ++n; return true; }); + g_assert_cmpuint(n,==,0); + } +} + + + +static void +test_mu_contacts_cache_sort() +{ + auto result_chars = [](const Mu::ContactsCache& ccache)->std::string { + std::string str; + if (g_test_verbose()) + fmt::print("contacts-cache:\n"); + + ccache.for_each([&](auto&& contact) { + if (g_test_verbose()) + fmt::print("\t- {}\n", contact.display_name()); + str += contact.name; + return true; + }); + return str; + }; + + const auto now{std::time({})}; + + // "first" means more relevant + + { /* recent messages, newer comes first */ + + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", now, true, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", now-1, true, 1000, 0}); + assert_equal(result_chars(ccache), "ab"); + } + + { /* non-recent messages, more frequent comes first */ + + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", now-2*RecentOffset, true, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", now-3*RecentOffset, true, 2000, 0}); + assert_equal(result_chars(ccache), "ba"); + } + + { /* personal comes first */ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", now-5*RecentOffset, true, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", now, false, 8000, 0}); + assert_equal(result_chars(ccache), "ab"); + } + + { /* if all else fails, reverse-alphabetically */ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", now, false, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", now, false, 1000, 0}); + g_assert_cmpuint(ccache.size(),==,2); + assert_equal(result_chars(ccache), "ba"); + } +} + +static void +test_mu_contacts_valid_address() +{ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + g_assert_true(ccache.is_valid("a@example.com")); + g_assert_false(ccache.is_valid("a***@@booa@example..com")); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/contacts-cache/base", test_mu_contacts_cache_base); + g_test_add_func("/contacts-cache/personal", test_mu_contacts_cache_personal); + g_test_add_func("/contacts-cache/ignored", test_mu_contacts_cache_ignored); + g_test_add_func("/contacts-cache/for-each", test_mu_contacts_cache_foreach); + g_test_add_func("/contacts-cache/sort", test_mu_contacts_cache_sort); + g_test_add_func("/contacts-cache/valid-address", test_mu_contacts_valid_address); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-contacts-cache.hh b/lib/mu-contacts-cache.hh new file mode 100644 index 0000000..d31c9dc --- /dev/null +++ b/lib/mu-contacts-cache.hh @@ -0,0 +1,171 @@ +/* +** Copyright (C) 2020-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_CONTACTS_CACHE_HH__ +#define __MU_CONTACTS_CACHE_HH__ + +#include <glib.h> +#include <time.h> +#include <memory> +#include <functional> +#include <chrono> +#include <string> +#include <time.h> +#include <inttypes.h> +#include <utils/mu-utils.hh> + +#include "mu-config.hh" +#include <message/mu-message.hh> + +namespace Mu { + +class ContactsCache { +public: + /** + * Construct a new ContactsCache object + * + * @param config db configuration database object + */ + ContactsCache(Config& config); + + /** + * DTOR + * + */ + ~ContactsCache(); + + /** + * Add a contact + * + * Invalid email address are not cached (but we log a warning); neither + * are "ignored" addresses (see --ignored-address in mu-init(1)) + * + * @param contact a Contact object + */ + void add(Contact&& contact); + + + /** + * Add a contacts sequence; this should be used for the contacts of a + * specific message, and determines if it is a "personal" message: + * if any of the contacts matches one of the personal addresses, + * any of the senders/recipients are considered "personal" + * + * Invalid email address are not cached (but we log a warning); neither + * are "ignored" addresses (see --ignored-address in mu-init(1)) + * + * @param contacts a Contact object sequence + * @param is_personal receives true if any of the contacts was personal; + * false otherwise + */ + void add(Contacts&& contacts, bool& is_personal); + void add(Contacts&& contacts) { + bool _ignore; + add(std::move(contacts), _ignore); + } + + /** + * Clear all contacts + * + */ + void clear(); + + /** + * Get the number of contacts + * + + * @return number of contacts + */ + std::size_t size() const; + + /** + * Are there no contacts? + * + * @return true or false + */ + bool empty() const { return size() == 0; } + + /** + * Serialize contact information. This all marks the data as + * non-dirty + */ + void serialize() const; + + /** + * Does this look like a 'personal' address? + * + * @param addr some e-mail address + * + * @return true or false + */ + bool is_personal(const std::string& addr) const; + + /** + * Does this look like an email-address that should be ignored? + * + * @param addr some e-mail address + * + * @return true or false + */ + bool is_ignored(const std::string& addr) const; + + /** + * Does this look like a valid email-address? + * + * @param addr some e-mail address + * + * @return true or false + */ + bool is_valid(const std::string& addr) const; + + /** + * Find a contact based on the email address. This is not safe, since + * the returned ptr can be invalidated at any time; only for unit-tests. + * + * @param email email address + * + * @return contact info, or {} if not found + */ + const Contact* _find(const std::string& email) const; + + /** + * Prototype for a callable that receives a contact + * + * @param contact some contact + * + * @return to get more contacts; false otherwise + */ + using EachContactFunc = std::function<bool(const Contact& contact_info)>; + + /** + * Invoke some callable for each contact, in _descending_ order of rank (i.e., the + * highest ranked contacts come first). + * + * @param each_contact function invoked for each contact + */ + void for_each(const EachContactFunc& each_contact) const; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; + +} // namespace Mu + +#endif /* __MU_CONTACTS_CACHE_HH__ */ diff --git a/lib/mu-indexer.cc b/lib/mu-indexer.cc new file mode 100644 index 0000000..e764933 --- /dev/null +++ b/lib/mu-indexer.cc @@ -0,0 +1,663 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-indexer.hh" + +#include <config.h> + +#include <atomic> +#include <algorithm> +#include <mutex> +#include <vector> +#include <thread> +#include <condition_variable> +#include <iostream> +#include <atomic> +#include <chrono> +using namespace std::chrono_literals; + +#include "mu-store.hh" + +#include "mu-scanner.hh" +#include "utils/mu-async-queue.hh" +#include "utils/mu-error.hh" + +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +struct IndexState { + enum State { Idle, + Scanning, + Finishing, + Cleaning + }; + static const char* name(State s) { + switch (s) { + case Idle: + return "idle"; + case Scanning: + return "scanning"; + case Finishing: + return "finishing"; + case Cleaning: + return "cleaning"; + default: + return "<error>"; + } + } + + bool operator==(State rhs) const { + return state_.load() == rhs; + } + bool operator!=(State rhs) const { + return state_.load() != rhs; + } + void change_to(State new_state) { + mu_debug("changing indexer state {}->{}", name((State)state_), + name((State)new_state)); + state_.store(new_state); + } + +private: + std::atomic<State> state_{Idle}; +}; + +struct Indexer::Private { + Private(Mu::Store& store) + : store_{store}, scanner_{store_.root_maildir(), + [this](auto&& path, + auto&& statbuf, auto&& info) { + return handler(path, statbuf, info); + }}, + max_message_size_{store_.config().get<Mu::Config::Id::MaxMessageSize>()}, + was_empty_{store.empty()} { + + mu_message("created indexer for {} -> " + "{} (batch-size: {}; was-empty: {}; ngrams: {})", + store.root_maildir(), store.path(), + store.config().get<Mu::Config::Id::BatchSize>(), + was_empty_, + store.config().get<Mu::Config::Id::SupportNgrams>()); + } + + ~Private() { + stop(); + } + + bool dir_predicate(const std::string& path, const struct dirent* dirent) const; + bool handler(const std::string& fullpath, struct stat* statbuf, Scanner::HandleType htype); + + void maybe_start_worker(); + void item_worker(); + void scan_worker(); + + bool add_message(const std::string& path); + + bool cleanup(); + bool start(const Indexer::Config& conf, bool block); + bool stop(); + + bool is_running() const { return state_ != IndexState::Idle; } + + Indexer::Config conf_; + Store& store_; + Scanner scanner_; + const size_t max_message_size_; + + ::time_t dirstamp_{}; + std::size_t max_workers_; + std::vector<std::thread> workers_; + std::thread scanner_worker_; + + struct WorkItem { + std::string full_path; + enum Type { + Dir, + File + }; + Type type; + }; + + AsyncQueue<WorkItem> todos_; + + Progress progress_{}; + IndexState state_{}; + std::mutex lock_, w_lock_; + std::atomic<time_t> completed_{}; + bool was_empty_{}; +}; + +bool +Indexer::Private::handler(const std::string& fullpath, struct stat* statbuf, + Scanner::HandleType htype) +{ + switch (htype) { + case Scanner::HandleType::EnterDir: + case Scanner::HandleType::EnterNewCur: { + if (fullpath.length() > MaxTermLength) { + // currently the db uses the path as a key, and + // therefore it cannot be too long. We'd get an error + // later anyway but for now it's useful for surviving + // circular symlinks + mu_warning("'{}' is too long; ignore", fullpath); + return false; + } + + // in lazy-mode, we ignore this dir if its dirstamp suggest it + // is up-to-date (this is _not_ always true; hence we call it + // lazy-mode); only for actual message dirs, since the dir + // tstamps may not bubble up.U + dirstamp_ = store_.dirstamp(fullpath); + if (conf_.lazy_check && dirstamp_ >= statbuf->st_ctime && + htype == Scanner::HandleType::EnterNewCur) { + mu_debug("skip {} (seems up-to-date: {:%FT%T} >= {:%FT%T})", + fullpath, mu_time(dirstamp_), mu_time(statbuf->st_ctime)); + return false; + } + + // don't index dirs with '.noindex' + auto noindex = ::access((fullpath + "/.noindex").c_str(), F_OK) == 0; + if (noindex) { + mu_debug("skip {} (has .noindex)", fullpath); + return false; // don't descend into this dir. + } + + // don't index dirs with '.noupdate', unless we do a full + // (re)index. + if (!conf_.ignore_noupdate) { + auto noupdate = ::access((fullpath + "/.noupdate").c_str(), F_OK) == 0; + if (noupdate) { + mu_debug("skip {} (has .noupdate)", fullpath); + return false; + } + } + + mu_debug("checked {}", fullpath); + return true; + } + case Scanner::HandleType::LeaveDir: { + todos_.push({fullpath, WorkItem::Type::Dir}); + return true; + } + + case Scanner::HandleType::File: { + ++progress_.checked; + + if ((size_t)statbuf->st_size > max_message_size_) { + mu_debug("skip {} (too big: {} bytes)", fullpath, statbuf->st_size); + return false; + } + + // if the message is not in the db yet, or not up-to-date, queue + // it for updating/inserting. + if (statbuf->st_ctime <= dirstamp_ && store_.contains_message(fullpath)) + return false; + + // push the remaining messages to our "todo" queue for + // (re)parsing and adding/updating to the database. + todos_.push({fullpath, WorkItem::Type::File}); + return true; + } + default: + g_return_val_if_reached(false); + return false; + } +} + +void +Indexer::Private::maybe_start_worker() +{ + std::lock_guard lock{w_lock_}; + + if (todos_.size() > workers_.size() && workers_.size() < max_workers_) { + workers_.emplace_back(std::thread([this] { item_worker(); })); + mu_debug("added worker {}", workers_.size()); + } +} + +bool +Indexer::Private::add_message(const std::string& path) +{ + /* + * Having the lock here makes things a _lot_ slower. + * + * The reason for having the lock is some helgrind warnings; + * but it believed those are _false alarms_ + * https://gitlab.gnome.org/GNOME/glib/-/issues/2662 + * + * For now, set the lock as we were seeing some db corruption. + */ + std::unique_lock lock{w_lock_}; + auto msg{Message::make_from_path(path, store_.message_options())}; + if (!msg) { + mu_warning("failed to create message from {}: {}", path, msg.error().what()); + return false; + } + // if the store was empty, we know that the message is completely new + // and can use the fast path (Xapians 'add_document' rather tahn + // 'replace_document) + auto res = store_.add_message(msg.value(), was_empty_); + if (!res) { + mu_warning("failed to add message @ {}: {}", path, res.error().what()); + return false; + } + + return true; +} + +void +Indexer::Private::item_worker() +{ + WorkItem item; + + mu_debug("started worker"); + + while (state_ == IndexState::Scanning) { + if (!todos_.pop(item, 250ms)) + continue; + try { + switch (item.type) { + case WorkItem::Type::File: { + if (G_LIKELY(add_message(item.full_path))) + ++progress_.updated; + } break; + case WorkItem::Type::Dir: + store_.set_dirstamp(item.full_path, ::time(NULL)); + break; + default: + g_warn_if_reached(); + break; + } + } catch (const Mu::Error& er) { + mu_warning("error adding message @ {}: {}", item.full_path, er.what()); + } + + maybe_start_worker(); + std::this_thread::yield(); + } +} + +bool +Indexer::Private::cleanup() +{ + mu_debug("starting cleanup"); + + size_t n{}; + std::vector<Store::Id> orphans; // store messages without files. + store_.for_each_message_path([&](Store::Id id, const std::string& path) { + ++n; + if (::access(path.c_str(), R_OK) != 0) { + mu_debug("cannot read {} (id={}); queuing for removal from store", + path, id); + orphans.emplace_back(id); + } + + return state_ == IndexState::Cleaning; + }); + + if (orphans.empty()) + mu_debug("nothing to clean up"); + else { + mu_debug("removing {} stale message(s) from store", orphans.size()); + store_.remove_messages(orphans); + progress_.removed += orphans.size(); + } + + return true; +} + +void +Indexer::Private::scan_worker() +{ + XapianDb::Transaction tx{store_.xapian_db()}; // RAII + + progress_.reset(); + if (conf_.scan) { + mu_debug("starting scanner"); + if (!scanner_.start()) { // blocks. + mu_warning("failed to start scanner"); + state_.change_to(IndexState::Idle); + return; + } + mu_debug("scanner finished with {} file(s) in queue", todos_.size()); + } + + // now there may still be messages in the work queue... + // finish those; this is a bit ugly; perhaps we should + // handle SIGTERM etc. + + if (!todos_.empty()) { + const auto workers_size = std::invoke([this] { + std::lock_guard lock{w_lock_}; + return workers_.size(); + }); + mu_debug("process {} remaining message(s) with {} worker(s)", + todos_.size(), workers_size); + while (!todos_.empty()) + std::this_thread::sleep_for(100ms); + } + // and let the worker finish their work. + state_.change_to(IndexState::Finishing); + for (auto&& w : workers_) + if (w.joinable()) + w.join(); + + if (conf_.cleanup) { + mu_debug("starting cleanup"); + state_.change_to(IndexState::Cleaning); + cleanup(); + mu_debug("cleanup finished"); + } + + completed_ = ::time({}); + store_.config().set<Mu::Config::Id::LastIndex>(completed_); + state_.change_to(IndexState::Idle); +} + +bool +Indexer::Private::start(const Indexer::Config& conf, bool block) +{ + stop(); + + conf_ = conf; + if (conf_.max_threads == 0) { + /* benchmarking suggests that ~4 threads is the fastest (the + * real bottleneck is the database, so adding more threads just + * slows things down) + */ + max_workers_ = std::min(4U, std::thread::hardware_concurrency()); + } else + max_workers_ = conf.max_threads; + + if (store_.empty() && conf_.lazy_check) { + mu_debug("turn off lazy check since we have an empty store"); + conf_.lazy_check = false; + } + + mu_debug("starting indexer with <= {} worker thread(s)", max_workers_); + mu_debug("indexing: {}; clean-up: {}", conf_.scan ? "yes" : "no", + conf_.cleanup ? "yes" : "no"); + + state_.change_to(IndexState::Scanning); + /* kick off the first worker, which will spawn more if needed. */ + workers_.emplace_back(std::thread([this] { item_worker(); })); + /* kick the disk-scanner thread */ + scanner_worker_ = std::thread([this] { scan_worker(); }); + + mu_debug("started indexer in {}-mode", block ? "blocking" : "non-blocking"); + if (block) { + while(is_running()) { + using namespace std::chrono_literals; + std::this_thread::sleep_for(100ms); + } + } + + return true; +} + +bool +Indexer::Private::stop() +{ + scanner_.stop(); + + todos_.clear(); + if (scanner_worker_.joinable()) + scanner_worker_.join(); + + state_.change_to(IndexState::Idle); + for (auto&& w : workers_) + if (w.joinable()) + w.join(); + workers_.clear(); + + return true; +} + +Indexer::Indexer(Store& store) + : priv_{std::make_unique<Private>(store)} +{} + +Indexer::~Indexer() = default; + +bool +Indexer::start(const Indexer::Config& conf, bool block) +{ + const auto mdir{priv_->store_.root_maildir()}; + if (G_UNLIKELY(access(mdir.c_str(), R_OK) != 0)) { + mu_critical("'{}' is not readable: {}", mdir, g_strerror(errno)); + return false; + } + + std::lock_guard lock(priv_->lock_); + if (is_running()) + return true; + + return priv_->start(conf, block); +} + +bool +Indexer::stop() +{ + std::lock_guard lock{priv_->lock_}; + + if (!is_running()) + return true; + + mu_debug("stopping indexer"); + return priv_->stop(); +} + +bool +Indexer::is_running() const +{ + return priv_->is_running(); +} + +const Indexer::Progress& +Indexer::progress() const +{ + priv_->progress_.running = priv_->state_ == IndexState::Idle ? false : true; + + return priv_->progress_; +} + +::time_t +Indexer::completed() const +{ + return priv_->completed_; +} + + +#if BUILD_TESTS +#include "mu-test-utils.hh" + +static void +test_index_basic() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + + Indexer& idx{store->indexer()}; + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + g_assert_cmpuint(idx.completed(),==, 0); + + const auto& prog{idx.progress()}; + g_assert_false(prog.running); + g_assert_cmpuint(prog.checked,==, 0); + g_assert_cmpuint(prog.updated,==, 0); + g_assert_cmpuint(prog.removed,==, 0); + + Indexer::Config conf{}; + conf.ignore_noupdate = true; + + { + const auto start{time({})}; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + + g_assert_cmpuint(idx.completed() - start, <, 5); + + g_assert_false(prog.running); + g_assert_cmpuint(prog.checked,==, 14); + g_assert_cmpuint(prog.updated,==, 14); + g_assert_cmpuint(prog.removed,==, 0); + + g_assert_cmpuint(store->size(),==,14); + } + + conf.lazy_check = true; + conf.max_threads = 1; + conf.ignore_noupdate = false; + + { + const auto start{time({})}; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + + g_assert_cmpuint(idx.completed() - start, <, 3); + + g_assert_false(prog.running); + g_assert_cmpuint(prog.checked,==, 0); + g_assert_cmpuint(prog.updated,==, 0); + g_assert_cmpuint(prog.removed,==, 0); + + g_assert_cmpuint(store->size(),==, 14); + } +} + + +static void +test_index_lazy() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + Indexer& idx{store->indexer()}; + + Indexer::Config conf{}; + conf.lazy_check = true; + conf.ignore_noupdate = false; + + const auto start{time({})}; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + + g_assert_cmpuint(idx.completed() - start, <, 3); + + const auto& prog{idx.progress()}; + g_assert_false(prog.running); + g_assert_cmpuint(prog.checked,==, 6); + g_assert_cmpuint(prog.updated,==, 6); + g_assert_cmpuint(prog.removed,==, 0); + + g_assert_cmpuint(store->size(),==, 6); +} + +static void +test_index_cleanup() +{ + allow_warnings(); + + TempDir tdir; + auto mdir = join_paths(tdir.path(), "Test"); + { + auto res = run_command({"cp", "-r", MU_TESTMAILDIR2, mdir}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==, 0); + } + + auto store = Store::make_new(tdir.path(), mdir); + assert_valid_result(store); + g_assert_true(store->empty()); + Indexer& idx{store->indexer()}; + + Indexer::Config conf{}; + conf.ignore_noupdate = true; + + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + g_assert_cmpuint(store->size(),==, 14); + + // remove a message + { + auto mpath = join_paths(mdir, "bar", "cur", "mail6"); + auto res = run_command({"rm", mpath}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==, 0); + } + + // no cleanup, # stays the same + conf.cleanup = false; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + g_assert_cmpuint(store->size(),==, 14); + + // cleanup, message is gone from store. + conf.cleanup = true; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + g_assert_cmpuint(store->size(),==, 13); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/index/basic", test_index_basic); + g_test_add_func("/index/lazy", test_index_lazy); + g_test_add_func("/index/cleanup", test_index_cleanup); + + return g_test_run(); + +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-indexer.hh b/lib/mu-indexer.hh new file mode 100644 index 0000000..3ea1fb6 --- /dev/null +++ b/lib/mu-indexer.hh @@ -0,0 +1,122 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_INDEXER_HH__ +#define MU_INDEXER_HH__ + +#include <atomic> +#include <memory> +#include <chrono> + +namespace Mu { + +class Store; + +/// An object abstracting the index process. +class Indexer { +public: + /** + * Construct an indexer object + * + * @param store the message store to use + */ + Indexer(Store& store); + + /** + * DTOR + */ + ~Indexer(); + + /// A configuration object for the indexer + struct Config { + bool scan{true}; + /**< scan for new messages */ + bool cleanup{true}; + /**< clean messages no longer in the file system */ + size_t max_threads{}; + /**< maximum # of threads to use */ + bool ignore_noupdate{}; + /**< ignore .noupdate files */ + bool lazy_check{}; + /**< whether to skip directories that don't have a changed + * mtime */ + }; + + /** + * Start indexing. If already underway, do nothing. This returns + * immediately after starting, with the work being done in the + * background, unless blocking = true + * + * @param conf a configuration object + * + * @return true if starting worked or an indexing process was already + * underway; false otherwise. + * + */ + bool start(const Config& conf, bool block=false); + + /** + * Stop indexing. If not indexing, do nothing. + * + * @return true if we stopped indexing, or indexing was not underway; false otherwise. + */ + bool stop(); + + /** + * Is an indexing process running? + * + * @return true or false. + */ + bool is_running() const; + + // Object describing current progress + struct Progress { + void reset() { + running = false; + checked = updated = removed = 0; + } + std::atomic<bool> running{}; /**< Is an index operation in progress? */ + std::atomic<size_t> checked{}; /**< Number of messages checked for changes */ + std::atomic<size_t> updated{}; /**< Number of messages (re)parsed/added/updated */ + std::atomic<size_t> removed{}; /**< Number of message removed from store */ + }; + + /** + * Get an object describing the current progress. The progress object + * describes the most recent indexing job, and is reset upon a fresh + * start(). + * + * @return a progress object. + */ + const Progress& progress() const; + + /** + * Last time indexing was completed. + * + * @return the time or 0 + */ + ::time_t completed() const; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; + +} // namespace Mu +#endif /* MU_INDEXER_HH__ */ diff --git a/lib/mu-maildir.cc b/lib/mu-maildir.cc new file mode 100644 index 0000000..5166b17 --- /dev/null +++ b/lib/mu-maildir.cc @@ -0,0 +1,455 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to 59the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <string> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include <fcntl.h> +#include <stdlib.h> + +#include <string.h> +#include <errno.h> +#include <glib/gprintf.h> +#include <gio/gio.h> + +#include "glibconfig.h" +#include "mu-maildir.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +#define MU_MAILDIR_NOINDEX_FILE ".noindex" +#define MU_MAILDIR_NOUPDATE_FILE ".noupdate" + +/* On Linux (and some BSD), we have entry->d_type, but some file + * systems (XFS, ReiserFS) do not support it, and set it DT_UNKNOWN. + * On other OSs, notably Solaris, entry->d_type is not present at all. + * For these cases, we use lstat (in get_dtype) as a slower fallback, + * and return it in the d_type parameter + */ +static unsigned char +get_dtype(struct dirent* dentry, const std::string& path, bool use_lstat) +{ +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + + if (dentry->d_type == DT_UNKNOWN) + goto slowpath; + if (dentry->d_type == DT_LNK && !use_lstat) + goto slowpath; + + return dentry->d_type; /* fastpath */ + +slowpath: +#endif /*HAVE_STRUCT_DIRENT_D_TYPE*/ + + return determine_dtype(path, use_lstat); +} + +static Mu::Result<void> +create_maildir(const std::string& path, mode_t mode) +{ + if (path.empty()) + return Err(Error{Error::Code::File, "path must not be empty"}); + + std::array<std::string,3> subdirs = {"new", "cur", "tmp"}; + for (auto&& subdir: subdirs) { + + const auto fullpath{join_paths(path, subdir)}; + + /* if subdir already exists, don't try to re-create + * it */ + if (check_dir(fullpath, true/*readable*/, true/*writable*/)) + continue; + + int rv{g_mkdir_with_parents(fullpath.c_str(), static_cast<int>(mode))}; + + /* note, g_mkdir_with_parents won't detect an error if + * there's already such a dir, but with the wrong + * permissions; so we need to check */ + if (rv != 0 || !check_dir(fullpath, true/*readable*/, true/*writable*/)) + return Err(Error{Error::Code::File, + "creating dir failed for {}: {}", + fullpath, g_strerror(errno)}); + } + + return Ok(); +} + +static Mu::Result<void> /* create a noindex file if requested */ +create_noindex(const std::string& path) +{ + const auto noindexpath{join_paths(path, MU_MAILDIR_NOINDEX_FILE)}; + + /* note, if the 'close' failed, creation may still have succeeded...*/ + int fd = ::creat(noindexpath.c_str(), 0644); + if (fd < 0 || ::close(fd) != 0) + return Err(Error{Error::Code::File, + "error creating .noindex: {}", g_strerror(errno)}); + else + return Ok(); +} + +Mu::Result<void> +Mu::maildir_mkdir(const std::string& path, mode_t mode, bool noindex) +{ + if (auto&& created{create_maildir(path, mode)}; !created) + return created; // fail. + else if (!noindex) + return Ok(); + + if (auto&& created{create_noindex(path)}; !created) + return created; //fail + + return Ok(); +} + +/* determine whether the source message is in 'new' or in 'cur'; + * we ignore messages in 'tmp' for obvious reasons */ +static Mu::Result<void> +check_subdir(const std::string& src, bool& in_cur) +{ + char *srcpath{g_path_get_dirname(src.c_str())}; + + bool invalid{}; + if (g_str_has_suffix(srcpath, "cur")) + in_cur = true; + else if (g_str_has_suffix(srcpath, "new")) + in_cur = false; + else + invalid = true; + + g_free(srcpath); + + if (invalid) + return Err(Error{Error::Code::File, "invalid source message '{}'", src}); + else + return Ok(); +} + +static Mu::Result<std::string> +get_target_fullpath(const std::string& src, const std::string& targetpath, + bool unique_names) +{ + bool in_cur{}; + if (auto&& res = check_subdir(src, in_cur); !res) + return Err(std::move(res.error())); + + const auto srcfile{basename(src)}; + + /* create target-path; note: make the filename *cough* unique by + * including a hash of the srcname in the targetname. This helps if + * there are copies of a message (which all have the same basename) + */ + if (unique_names) + return join_paths(targetpath, in_cur ? "cur" : "new", + mu_format("{:08x}-{}", g_str_hash(src.c_str()), srcfile)); + else + return join_paths(targetpath, in_cur ? "cur" : "new", srcfile.c_str()); +} + +Result<void> +Mu::maildir_link(const std::string& src, const std::string& targetpath, + bool unique_names) +{ + auto path_res{get_target_fullpath(src, targetpath, unique_names)}; + if (!path_res) + return Err(std::move(path_res.error())); + + auto rv{::symlink(src.c_str(), path_res->c_str())}; + if (rv != 0) + return Err(Error{Error::Code::File, + "error creating link {} => {}: {}", + *path_res, src, g_strerror(errno)}); + + return Ok(); +} + +static bool +clear_links(const std::string& path, DIR* dir) +{ + bool res; + struct dirent* dentry; + + res = true; + errno = 0; + + while ((dentry = ::readdir(dir))) { + + if (dentry->d_name[0] == '.') + continue; /* ignore .,.. other dotdirs */ + + const auto fullpath{join_paths(path, dentry->d_name)}; + const auto d_type = get_dtype(dentry, fullpath.c_str(), + true/*lstat*/); + switch(d_type) { + case DT_LNK: + if (::unlink(fullpath.c_str()) != 0) { + /* LCOV_EXCL_START*/ + mu_warning("error unlinking {}: {}", fullpath, g_strerror(errno)); + res = false; + /* LCOV_EXCL_STOP*/ + } else + mu_debug("unlinked linksdir {}", fullpath); + break; + case DT_DIR: { + DIR* subdir{::opendir(fullpath.c_str())}; + /* LCOV_EXCL_START*/ + if (!subdir) { + mu_warning("error opening dir {}: {}", fullpath, g_strerror(errno)); + res = false; + } + if (!clear_links(fullpath, subdir)) + res = false; + /* LCOV_EXCL_STOP*/ + ::closedir(subdir); + } break; + default: + break; + } + } + + return res; +} + +Mu::Result<void> +Mu::maildir_clear_links(const std::string& path) +{ + const auto dir{::opendir(path.c_str())}; + if (!dir) + return Err(Error{Error::Code::File, "failed to open {}: {}", + path, g_strerror(errno)}); + + clear_links(path, dir); + ::closedir(dir); + + return Ok(); +} + +/* LCOV_EXCL_START*/ +static Mu::Result<void> +msg_move_verify(const std::string& src, const std::string& dst) +{ + /* double check -- is the target really there? */ + if (::access(dst.c_str(), F_OK) != 0) + return Err(Error{Error::Code::File, + "can't find target ({}->{})", src, dst}); + + if (::access(src.c_str(), F_OK) == 0) { + if (src == dst) { + mu_warning("moved {} to itself", src); + } + /* this could happen if some other tool (for mail syncing) is + * interfering */ + mu_debug("source is still there ({}->{})", src, dst); + } + + mu_debug("moved {} -> {}", src, dst); + + return Ok(); +} +/* LCOV_EXCL_STOP*/ + +/* LCOV_EXCL_START*/ +// don't use this right now, since it gives as (false alarm) +// valgrind warning in tests +/* use GIO to move files; this is slower than rename() so only use + * this when needed: when moving across filesystems */ +G_GNUC_UNUSED static Mu::Result<void> +msg_move_g_file(const std::string& src, const std::string& dst) +{ + GFile *srcfile{g_file_new_for_path(src.c_str())}; + GFile *dstfile{g_file_new_for_path(dst.c_str())}; + + GError* err{}; + auto res = g_file_move(srcfile, dstfile, + G_FILE_COPY_OVERWRITE, + NULL, NULL, NULL, &err); + g_clear_object(&srcfile); + g_clear_object(&dstfile); + + if (res) + return Ok(); + else + return Err(Error::Code::File, &err, "error moving {} -> {}", src, dst); +} +/* LCOV_EXCL_STOP*/ + +/* use mv to move files; this is slower than rename() so only use this when + * needed: when moving across filesystems */ +G_GNUC_UNUSED static Mu::Result<void> +msg_move_mv_file(const std::string& src, const std::string& dst) +{ + if (auto res{run_command0({"/bin/mv", src, dst})}; !res) + return Err(Error::Code::File, "error moving {}->{}; err={}", src, dst, res.error()); + else + return Ok(); +} + +Mu::Result<void> +Mu::maildir_move_message(const std::string& oldpath, + const std::string& newpath, + bool assume_remote) +{ + mu_debug("moving {} --> {} (assume-remote:{})", oldpath, newpath, assume_remote); + + if (::access(oldpath.c_str(), R_OK) != 0) + return Err(Error{Error::Code::File, "cannot read {}", oldpath}); + + if (oldpath == newpath) + return Ok(); // nothing to do. + + if (!assume_remote) { /* for testing */ + + if (::rename(oldpath.c_str(), newpath.c_str()) == 0) /* seems it worked; double-check */ + return msg_move_verify(oldpath, newpath); + /* LCOV_EXCL_START*/ + if (errno != EXDEV) /* some unrecoverable error occurred */ + return Err(Error{Error::Code::File, "error moving {} -> {}: {}", + oldpath, newpath, strerror(errno)}); + /* LCOV_EXCL_STOP*/ + } + + /* the EXDEV / assume-remote case -- source and target live on different + * file systems + * + * we can choose either msg_move_gio_file or msg_move_mv_file; + * we use the latter for now, since the former gives some (false) + * valgrind alarms. + * */ + if (auto&& res{msg_move_mv_file(oldpath, newpath)}; !res) + return res; + else + return msg_move_verify(oldpath, newpath); +} + +static std::string +reinvent_filename_base() +{ + return mu_format("{}.{:08x}{:08x}.{}", ::time({}), + g_random_int(), g_get_monotonic_time(), g_get_host_name()); +} + +/** + * Determine the destination filename + * + * @param file a filename + * @param flags flags for the destination + * @param new_name whether to change the basename + * + * @return the destion filename. + */ +static std::string +determine_dst_filename(const std::string& file, Flags flags, + bool new_name) +{ + /* Recalculate a unique new base file name */ + auto&& parts{message_file_parts(file)}; + if (new_name) + parts.base = reinvent_filename_base(); + + /* for a New message, there are no flags etc.; so we only return the + * name sans suffix */ + if (any_of(flags & Flags::New)) + return std::move(parts.base); + + const auto flagstr{ + to_string( + flags_filter( + flags, MessageFlagCategory::Mailfile))}; + + return parts.base + parts.separator + "2," + flagstr; +} + + +/* + * sanity checks + */ +static Mu::Result<void> +check_determine_target_params (const std::string& old_path, + const std::string& root_maildir_path, + const std::string& target_maildir, + Flags newflags) +{ + if (!g_path_is_absolute(old_path.c_str())) + return Err(Error{Error::Code::File, + "old_path is not absolute ({})", old_path}); + + if (!g_path_is_absolute(root_maildir_path.c_str())) + return Err(Error{Error::Code::File, + "root maildir path is not absolute ({})", + root_maildir_path}); + + if (!target_maildir.empty() && target_maildir[0] != '/') + return Err(Error{Error::Code::File, + "target maildir must be empty or start with / ({})", + target_maildir}); + + if (old_path.find(root_maildir_path) != 0) + return Err(Error{Error::Code::File, + "old-path must be below root-maildir ({}) ({})", + old_path, root_maildir_path}); + + if (any_of(newflags & Flags::New) && newflags != Flags::New) + return Err(Error{Error::Code::File, + "if the New flag is specified, it must be the only flag"}); + return Ok(); +} + + +Mu::Result<std::string> +Mu::maildir_determine_target(const std::string& old_path, + const std::string& root_maildir_path, + const std::string& target_maildir, + Flags newflags, + bool new_name) +{ + newflags = flags_maildir_file(newflags); // filter out irrelevant flags. + + /* sanity checks */ + if (const auto checked{check_determine_target_params( + old_path, root_maildir_path, target_maildir, newflags)}; !checked) + return Err(Error{std::move(checked.error())}); + + /* + * this gets us the source maildir filesystem path, the directory + * in which new/ & cur/ lives, and the source file + */ + const auto src{base_message_dir_file(old_path)}; + if (!src) + return Err(src.error()); + const auto& [src_mdir, src_file, is_new] = *src; + + /* if target_mdir is empty, the src_dir does not change (though cur/ + * maybe become new or vice-versa) */ + const auto dst_mdir = target_maildir.empty() ? src_mdir : + join_paths(root_maildir_path, target_maildir); + + /* now calculate the message name (incl. its immediate parent dir) */ + const auto dst_file{determine_dst_filename(src_file, newflags, new_name)}; + + /* and the complete path name. */ + const std::string subdir{(none_of(newflags & Flags::New)) ? "cur" : "new"}; + + return join_paths(dst_mdir, subdir,dst_file); +} diff --git a/lib/mu-maildir.hh b/lib/mu-maildir.hh new file mode 100644 index 0000000..e9e4c75 --- /dev/null +++ b/lib/mu-maildir.hh @@ -0,0 +1,120 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MAILDIR_HH__ +#define MU_MAILDIR_HH__ + +#include <string> +#include <utils/mu-result.hh> + +#include <glib.h> +#include <time.h> +#include <sys/types.h> /* for mode_t */ +#include <message/mu-message.hh> + +namespace Mu { + +/** + * Create a new maildir. if parts of the maildir already exists, those will + * simply be ignored. + * + * IOW, if you try to create the same maildir twice, the second will simply be a + * no-op (without any errors). Note, if the function fails 'halfway', it will + * *not* try to remove the parts the were created. it *will* create any parent + * dirs that are not yet existent. + * + * @param path the path (missing components will be created, as in 'mkdir -p'). + * must be non-empty + * @param mode the file mode (e.g., 0755) + * @param noindex add a .noindex file to the maildir, so it will be excluded + * from indexing by 'mu index' + * + * @return a valid result (!!result) or an Error + */ +Result<void> maildir_mkdir(const std::string& path, mode_t mode=0700, + bool noindex=false); + +/** + * Create a symbolic link to a mail message + * + * @param src the full path to the source message + * @param targetpath the path to the target maildir; ie., *not* + * MyMaildir/cur, but just MyMaildir/. The function will figure out + * the correct subdir then. + * @param unique_names whether to create unique names; should be true unless + * for tests. + * + * @return a valid result (!!result) or an Error + */ +Result<void> maildir_link(const std::string& src, const std::string& targetpath, + bool unique_names=true); +/** + * Recursively delete all the symbolic links in a directory tree + * + * @param dir top dir + * + * @return a valid result (!!result) or an Error + */ +Result<void> maildir_clear_links(const std::string& dir); + +/** + * Move a message file to another maildir. If the target exists, it is overwritten. + * + * @param oldpath an absolute file system path to an existing message in an + * actual maildir + * @param newpath the absolute full path to the target file + * @param assume_remote assume the target is on a different file-system, + * and hence rename() won't work and we need another method + * + * @return a valid result or an Error + */ +Result<void> maildir_move_message(const std::string& oldpath, + const std::string& newpath, + bool assume_remote = false); + +/** + * Determine the target path for a to-be-moved message; i.e. this does not + * actually move the message, only calculate the path. + * + * @param old_path an absolute file system path to an existing message in an + * actual maildir + * @param root_maildir_path the absolute file system path under which + * all maildirs live. + * @param target_maildir the target maildir; note that this the base-level + * Maildir, ie. /home/user/Maildir/archive, and must _not_ include the + * 'cur' or 'new' part. Note that the target maildir must be on the + * same filesystem. Can be empty if the message should not be moved to + * a different maildir; note that this may still involve a + * move to another directory (say, from new/ to cur/) + * @param flags to set for the target (influences the filename, path). + * Any non-Maildir/File flags are ignored. + * @param new_name whether to change the basename of the file + * + * @return Full path name of the target file or an Error + */ +Result<std::string> +maildir_determine_target(const std::string& old_path, + const std::string& root_maildir_path, + const std::string& target_maildir, + Flags newflags, + bool new_name); + +} // namespace Mu + +#endif /*MU_MAILDIR_HH__*/ diff --git a/lib/mu-query-macros.cc b/lib/mu-query-macros.cc new file mode 100644 index 0000000..1fb682b --- /dev/null +++ b/lib/mu-query-macros.cc @@ -0,0 +1,160 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-macros.hh" + +#include <glib.h> +#include <unordered_map> + +#include "utils/mu-utils.hh" + +using namespace Mu; + +constexpr auto MU_BOOKMARK_GROUP = "mu"; + +struct QueryMacros::Private { + Private(const Config& conf): conf_{conf} {} + + + Result<void> import_key_file(GKeyFile *kfile); + + const Config& conf_; + std::unordered_map<std::string, std::string> macros_{}; +}; + +Result<void> +QueryMacros::Private::import_key_file(GKeyFile *kfile) +{ + if (!kfile) + return Err(Error::Code::InvalidArgument, "invalid key-file"); + + GError *err{}; + size_t num{}; + gchar **keys{g_key_file_get_keys(kfile, MU_BOOKMARK_GROUP, &num, &err)}; + if (!keys) + return Err(Error::Code::File, &err/*cons*/,"failed to read keys"); + + for (auto key = keys; key && *key; ++key) { + + auto rawval{g_key_file_get_string(kfile, MU_BOOKMARK_GROUP, *key, &err)}; + if (!rawval) { + g_strfreev(keys); + return Err(Error::Code::File, &err/*cons*/,"failed to read key '{}'", *key); + } + + auto val{to_string_gchar(std::move(rawval))}; + macros_.erase(val); // we want to replace + macros_.emplace(std::string(*key), std::move(val)); + ++num; + } + + g_strfreev(keys); + mu_debug("imported {} query macro(s); total {}", num, macros_.size()); + return Ok(); +} + +QueryMacros::QueryMacros(const Config& conf): + priv_{std::make_unique<Private>(conf)} {} + +QueryMacros::~QueryMacros() = default; + +Result<void> +QueryMacros::load_bookmarks(const std::string& path) +{ + GError *err{}; + GKeyFile *kfile{g_key_file_new()}; + if (!g_key_file_load_from_file(kfile, path.c_str(), G_KEY_FILE_NONE, &err)) { + g_key_file_unref(kfile); + return Err(Error::Code::File, &err/*cons*/, + "failed to read bookmarks from {}", path); + } + + auto&& res = priv_->import_key_file(kfile); + g_key_file_unref(kfile); + + return res; +} + +Option<std::string> +QueryMacros::find_macro(const std::string& name) const +{ + if (const auto it{priv_->macros_.find(name)}; it != priv_->macros_.end()) + return it->second; + else + return Nothing; +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" +#include "utils/mu-utils-file.hh" + +static void +test_bookmarks() +{ + MemDb db; + Config conf_db{db}; + QueryMacros qm{conf_db}; + + TempDir tdir{}; + const auto bmfile{join_paths(tdir.path(), "bookmarks.ini")}; + std::ofstream os{bmfile}; + + mu_println(os, "# test\n" + "[mu]\n" + "foo=subject:bar"); + os.close(); + + auto res = qm.load_bookmarks(bmfile); + assert_valid_result(res); + + assert_equal(qm.find_macro("foo").value_or(""), "subject:bar"); + assert_equal(qm.find_macro("bar").value_or("nope"), "nope"); +} + + +static void +test_bookmarks_fail() +{ + + MemDb db; + Config conf_db{db}; + QueryMacros qm{conf_db}; + + auto res = qm.load_bookmarks("/foo/bar/non-existent"); + g_assert_false(!!res); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/query/macros/bookmarks", test_bookmarks); + g_test_add_func("/query/macros/bookmarks-fail", test_bookmarks_fail); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query-macros.hh b/lib/mu-query-macros.hh new file mode 100644 index 0000000..1b62615 --- /dev/null +++ b/lib/mu-query-macros.hh @@ -0,0 +1,75 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#ifndef MU_QUERY_MACROS_HH__ +#define MU_QUERY_MACROS_HH__ + +#include <string> +#include <memory> + +#include <utils/mu-result.hh> +#include <utils/mu-option.hh> + +#include "mu-config.hh" + +namespace Mu { + +class QueryMacros{ +public: + /** + * Construct QueryMacros object + * + * @param conf config object ref + */ + QueryMacros(const Config& conf); + + /** + * DTOR + */ + ~QueryMacros(); + + /** + * Read bookmarks (ie. macros) from a bookmark-file + * + * @param bookmarks_file path to the bookmarks file + * + * @return Ok or some error + */ + Result<void> load_bookmarks(const std::string& bookmarks_file); + + + /** + * Find a macro (aka 'bookmark') by its name + * + * @param name the name of the bookmark + * + * @return the macro value or Nothing if not found + */ + Option<std::string> find_macro(const std::string& name) const; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; + + +} // namespace Mu + +#endif /* MU_QUERY_MACROS_HH__ */ diff --git a/lib/mu-query-match-deciders.cc b/lib/mu-query-match-deciders.cc new file mode 100644 index 0000000..999d609 --- /dev/null +++ b/lib/mu-query-match-deciders.cc @@ -0,0 +1,223 @@ +/* +** Copyright (C) 2020-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-match-deciders.hh" + +#include "mu-query-results.hh" +#include "utils/mu-option.hh" + +using namespace Mu; + + +// We use a MatchDecider to gather information about the matches, and decide +// whether to include them in the results. +// +// Note that to include the "related" messages, we need _two_ queries; the first +// one to get the initial matches (called the Leader-Query) and a Related-Query, +// to get the Leader matches + all messages that have a thread-id seen in the +// Leader matches. +// +// We use the MatchDecider to gather information and use it for both queries. + +struct MatchDecider : public Xapian::MatchDecider { + MatchDecider(QueryFlags qflags, DeciderInfo& info) : qflags_{qflags}, decider_info_{info} {} + /** + * Update the match structure with unreadable/duplicate flags + * + * @param doc a Xapian document. + * + * @return a new QueryMatch object + */ + QueryMatch make_query_match(const Xapian::Document& doc) const + { + QueryMatch qm{}; + + auto msgid{opt_string(doc, Field::Id::MessageId) + .value_or(*opt_string(doc, Field::Id::Path))}; + if (!decider_info_.message_ids.emplace(std::move(msgid)).second) + qm.flags |= QueryMatch::Flags::Duplicate; + + const auto path{opt_string(doc, Field::Id::Path)}; + if (!path || ::access(path->c_str(), R_OK) != 0) + qm.flags |= QueryMatch::Flags::Unreadable; + + return qm; + } + + /** + * Should this message be included in the results? + * + * @param qm a query match + * + * @return true or false + */ + bool should_include(const QueryMatch& qm) const + { + if (any_of(qflags_ & QueryFlags::SkipDuplicates) && + any_of(qm.flags & QueryMatch::Flags::Duplicate)) + return false; + + if (any_of(qflags_ & QueryFlags::SkipUnreadable) && + any_of(qm.flags & QueryMatch::Flags::Unreadable)) + return false; + + return true; + } + /** + * Gather thread ids from this match. + * + * @param doc the document (message) + * + */ + void gather_thread_ids(const Xapian::Document& doc) const + { + auto thread_id{opt_string(doc, Field::Id::ThreadId)}; + if (thread_id) + decider_info_.thread_ids.emplace(std::move(*thread_id)); + } + +protected: + const QueryFlags qflags_; + DeciderInfo& decider_info_; + +private: + Option<std::string> opt_string(const Xapian::Document& doc, Field::Id id) const noexcept { + const auto value_no{field_from_id(id).value_no()}; + std::string val = xapian_try([&] { return doc.get_value(value_no); }, std::string{""}); + if (val.empty()) + return Nothing; + else + return Some(std::move(val)); + } +}; + +struct MatchDeciderLeader final : public MatchDecider { + MatchDeciderLeader(QueryFlags qflags, DeciderInfo& info) : MatchDecider(qflags, info) {} + /** + * operator() + * + * This receives the documents considered during a Xapian query, and + * is to return either true (keep) or false (ignore) + * + * We use this to potentiallly avoid certain messages (documents): + * - with QueryFlags::SkipUnreadable this will return false for message + * that are not readable in the file-system + * - with QueryFlags::SkipDuplicates this will return false for messages + * whose message-id was seen before. + * + * Even if we do not skip these messages entirely, we remember whether + * they were unreadable/duplicate (in the QueryMatch::Flags), so we can + * quickly find that info when doing the second 'related' query. + * + * The "leader" query. Matches here get the Leader flag unless they are + * duplicates / unreadable. We check the duplicate/readable status + * regardless of whether SkipDuplicates/SkipUnreadable was passed + * (to gather that information); however those flags + * affect our true/false verdict. + * + * @param doc xapian document + * + * @return true or false + */ + bool operator()(const Xapian::Document& doc) const override { + // by definition, we haven't seen the docid before, + // so no need to search + auto it = decider_info_.matches.emplace(doc.get_docid(), make_query_match(doc)); + it.first->second.flags |= QueryMatch::Flags::Leader; + + return should_include(it.first->second); + } +}; + +std::unique_ptr<Xapian::MatchDecider> +Mu::make_leader_decider(QueryFlags qflags, DeciderInfo& info) +{ + return std::make_unique<MatchDeciderLeader>(qflags, info); +} + +struct MatchDeciderRelated final : public MatchDecider { + MatchDeciderRelated(QueryFlags qflags, DeciderInfo& info) : MatchDecider(qflags, info) {} + /** + * operator() + * + * This receives the documents considered during a Xapian query, and + * is to return either true (keep) or false (ignore) + * + * We use this to potentially avoid certain messages (documents): + * - with QueryFlags::SkipUnreadable this will return false for message + * that are not readable in the file-system + * - with QueryFlags::SkipDuplicates this will return false for messages + * whose message-id was seen before. + * + * Unlike in the "leader" decider (scroll up), we don't need to remember + * messages we won't include. + * + * @param doc xapian document + * + * @return true or false + */ + bool operator()(const Xapian::Document& doc) const override { + // we may have seen this match in the "Leader" query. + const auto it = decider_info_.matches.find(doc.get_docid()); + if (it != decider_info_.matches.end()) + return should_include(it->second); + + auto qm{make_query_match(doc)}; + if (should_include(qm)) { + qm.flags |= QueryMatch::Flags::Related; + decider_info_.matches.emplace(doc.get_docid(), std::move(qm)); + return true; + } else + return false; // nope. + } +}; + +std::unique_ptr<Xapian::MatchDecider> +Mu::make_related_decider(QueryFlags qflags, DeciderInfo& info) +{ + return std::make_unique<MatchDeciderRelated>(qflags, info); +} + +struct MatchDeciderThread final : public MatchDecider { + MatchDeciderThread(QueryFlags qflags, DeciderInfo& info) : MatchDecider{qflags, info} {} + /** + * operator() + * + * This receives the documents considered during a Xapian query, and + * is to return either true (keep) or false (ignore) + * + * Only include documents that earlier checks have decided to include. + * + * @param doc xapian document + * + * @return true or false + */ + bool operator()(const Xapian::Document& doc) const override { + // we may have seen this match in the "Leader" query, + // or in the second (unbuounded) related query; + const auto it{decider_info_.matches.find(doc.get_docid())}; + return it != decider_info_.matches.end() && !it->second.thread_path.empty(); + } +}; + +std::unique_ptr<Xapian::MatchDecider> +Mu::make_thread_decider(QueryFlags qflags, DeciderInfo& info) +{ + return std::make_unique<MatchDeciderThread>(qflags, info); +} diff --git a/lib/mu-query-match-deciders.hh b/lib/mu-query-match-deciders.hh new file mode 100644 index 0000000..bd19605 --- /dev/null +++ b/lib/mu-query-match-deciders.hh @@ -0,0 +1,76 @@ +/* +** Copyright (C) 2021-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_QUERY_MATCH_DECIDERS_HH__ +#define MU_QUERY_MATCH_DECIDERS_HH__ + +#include <unordered_set> +#include <unordered_map> +#include <memory> + +#include "mu-xapian-db.hh" + +#include "mu-query-results.hh" + +namespace Mu { +using StringSet = std::unordered_set<std::string>; + +struct DeciderInfo { + QueryMatches matches; + StringSet thread_ids; + StringSet message_ids; +}; + +/** + * Make a "leader" decider, that is, a MatchDecider for either a singular or the + * first query in the leader/related pair of queries. Gather information for + * threading, and the subsequent "related" query. + * + * @param qflags query flags + * @param match_info receives information about the matches. + * + * @return a unique_ptr to a match decider. + */ +std::unique_ptr<Xapian::MatchDecider> make_leader_decider(QueryFlags qflags, DeciderInfo& info); + +/** + * Make a "related" decider, that is, a MatchDecider for the second query + * in the leader/related pair of queries. + * + * @param qflags query flags + * @param match_info receives information about the matches. + * + * @return a unique_ptr to a match decider. + */ +std::unique_ptr<Xapian::MatchDecider> make_related_decider(QueryFlags qflags, DeciderInfo& info); + +/** + * Make a "thread" decider, that is, a MatchDecider that removes all but the + * document excepts for the ones found during initial/related searches. + * + * @param qflags query flags + * @param match_info receives information about the matches. + * + * @return a unique_ptr to a match decider. + */ +std::unique_ptr<Xapian::MatchDecider> make_thread_decider(QueryFlags qflags, DeciderInfo& info); + +} // namespace Mu + +#endif /* MU_QUERY_MATCH_DECIDERS_HH__ */ diff --git a/lib/mu-query-parser.cc b/lib/mu-query-parser.cc new file mode 100644 index 0000000..87242dd --- /dev/null +++ b/lib/mu-query-parser.cc @@ -0,0 +1,485 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-parser.hh" + +#include <string_view> +#include <variant> +#include <type_traits> +#include <iostream> + +#include "utils/mu-utils.hh" +#include "utils/mu-sexp.hh" +#include "utils/mu-option.hh" +#include <glib.h> +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +// Sexp extensions... +static Sexp& +prepend(Sexp& s, Sexp&& e) +{ + s.list().insert(s.list().begin(), std::move(e)); + return s; +} + +static Option<Sexp&> +second(Sexp& s) +{ + if (s.listp() && !s.empty() && s.cbegin() + 1 != s.cend()) + return *(s.begin()+1); + else + return Nothing; +} + + +static bool +looks_like_matcher(const Sexp& sexp) +{ + // all the "terminal values" (from the Mu parser's pov) + const std::array<Sexp::Symbol, 5> value_syms = { + placeholder_sym, phrase_sym, regex_sym, range_sym, wildcard_sym + }; + + if (!sexp.listp() || sexp.empty() || !sexp.front().symbolp()) + return false; + + const auto symbol{sexp.front().symbol()}; + if (seq_some(value_syms, [&](auto &&sym) { return symbol == sym; })) + return true; + else if (!!field_from_name(symbol.name) || field_is_combi(symbol.name)) + return true; + else + return false; +} + +struct ParseContext { + bool expand; + std::vector<std::string> warnings; +}; + + + + +/** + * Indexable fields become _phrase_ fields if they contain + * wordbreakable data; + * + * @param field + * @param val + * + * @return + */ +static Option<Sexp> +phrasify(const Field& field, const Sexp& val) +{ + if (!field.is_phrasable_term() || !val.stringp()) + return Nothing; // nothing to phrasify + + auto words{utf8_wordbreak(val.string())}; + if (words.find(' ') == std::string::npos) + return Nothing; // nothing to phrasify + + auto phrase = Sexp { + Sexp::Symbol{field.name}, + Sexp{phrase_sym, Sexp{std::move(words)}}}; + + // if the field both a normal term & phrasable, match both + // if they are different + if (val.string() != words) + return Sexp{or_sym, + Sexp {Sexp::Symbol{field.name}, Sexp(val.string())}, + std::move(phrase)}; + else + return phrase; +} + + +/* + * Grammar + * + * query -> factor { (<OR> | <XOR>) factor } + * factor -> unit { [<AND>] unit } + * unit -> matcher | <NOT> query | <(> query <)> + * matcher + */ + +static Sexp query(Sexp& tokens, ParseContext& ctx); + + +static Sexp +matcher(Sexp& tokens, ParseContext& ctx) +{ + if (tokens.empty()) + return {}; + + auto val{*tokens.head()}; + tokens.pop_front(); + /* special case: if we find some non-matcher type here, we need to second-guess the token */ + if (!looks_like_matcher(val)) + val = Sexp{placeholder_sym, val.symbol().name}; + + const auto fieldsym{val.front().symbol()}; + + // Note the _expand_ case is what we use when processing the query 'for real'; + // the non-expand case is only to have a bit more human-readable Sexp for use + // mu find's '--analyze' + // + // Re: phrase-fields We map something like 'subject:hello-world' + // to + // (or (subject "hello-world" (subject (phrase "hello world")))) + + if (ctx.expand) { /* should we expand meta-fields? */ + auto fields = fields_from_name(fieldsym == placeholder_sym ? "" : fieldsym.name); + if (!fields.empty()) { + Sexp vals{}; + vals.add(or_sym); + for (auto&& field: fields) + if (auto&& phrase{phrasify(field, *second(val))}; phrase) + vals.add(std::move(*phrase)); + else + vals.add(Sexp{Sexp::Symbol{field.name}, Sexp{*second(val)}}); + val = std::move(vals); + } + + } + + if (auto&& field{field_from_name(fieldsym.name)}; field) { + if (auto&& phrase(phrasify(*field, *second(val))); phrase) + val = std::move(*phrase); + } + + return val; +} + +static Sexp +unit(Sexp& tokens, ParseContext& ctx) +{ + if (tokens.head_symbolp(not_sym)) { /* NOT */ + tokens.pop_front(); + Sexp sub{unit(tokens, ctx)}; + + /* special case: interpret "not" as a matcher instead; */ + if (sub.empty()) + return matcher(prepend(tokens, Sexp{placeholder_sym, not_sym.name}), ctx); + + /* we try to optimize: double negations are removed */ + if (sub.head_symbolp(not_sym)) + return *second(sub); + else + return Sexp(not_sym, std::move(sub)); + + } else if (tokens.head_symbolp(open_sym)) { /* ( sub) */ + tokens.pop_front(); + Sexp sub{query(tokens, ctx)}; + if (tokens.head_symbolp(close_sym)) + tokens.pop_front(); + else { + //g_warning("expected <)>"); + } + return sub; + } + + /* matcher */ + return matcher(tokens, ctx); +} + + +static Sexp +factor(Sexp& tokens, ParseContext& ctx) +{ + Sexp un = unit(tokens, ctx); + + /* query 'a b' is to be interpreted as 'a AND b'; + * + * we need an implicit AND if the head symbol is either + * a matcher (value) or the start of a sub-expression */ + auto implicit_and = [&]() { + if (tokens.head_symbolp(open_sym)) + return true; + else if (tokens.head_symbolp(not_sym)) // turn a lone 'not' -> 'and not' + return true; + else if (auto&& head{tokens.head()}; head) + return looks_like_matcher(*head); + else + return false; + }; + + Sexp uns; + while (true) { + if (tokens.head_symbolp(and_sym)) + tokens.pop_front(); + else if (!implicit_and()) + break; + + if (auto&& un2 = unit(tokens, ctx); !un2.empty()) + uns.add(std::move(un2)); + else + break; + } + + if (!uns.empty()) { + un = Sexp{and_sym, std::move(un)}; + un.add_list(std::move(uns)); + } + + return un; +} + +static Sexp +query(Sexp& tokens, ParseContext& ctx) +{ + /* note: we flatten (or (or ( or ...)) etc. here; + * for optimization (since Xapian likes flat trees) */ + + Sexp fact = factor(tokens, ctx); + Sexp or_factors, xor_factors; + while (true) { + auto factors = std::invoke([&]()->Option<Sexp&> { + + if (tokens.head_symbolp(or_sym)) + return or_factors; + else if (tokens.head_symbolp(xor_sym)) + return xor_factors; + else + return Nothing; + }); + + if (!factors) + break; + + tokens.pop_front(); + factors->add(factor(tokens, ctx)); + } + + // a bit clumsy... + + if (!or_factors.empty() && xor_factors.empty()) { + fact = Sexp{or_sym, std::move(fact)}; + fact.add_list(std::move(or_factors)); + } else if (or_factors.empty() && !xor_factors.empty()) { + fact = Sexp{xor_sym, std::move(fact)}; + fact.add_list(std::move(xor_factors)); + } else if (!or_factors.empty() && !xor_factors.empty()) { + fact = Sexp{or_sym, std::move(fact)}; + fact.add_list(std::move(or_factors)); + prepend(xor_factors, xor_sym); + fact.add(std::move(xor_factors)); + } + + return fact; +} + +Sexp +Mu::parse_query(const std::string& expr, bool expand) +{ + ParseContext context; + context.expand = expand; + + if (auto&& items = process_query(expr); !items.listp()) + throw std::runtime_error("tokens must be a list-sexp"); + else + return query(items, context); +} + + +#if defined(BUILD_PARSE_QUERY)||defined(BUILD_PARSE_QUERY_EXPAND) +int +main (int argc, char *argv[]) +{ + if (argc < 2) { + mu_printerrln("expected: {} <query>", argv[0]); + return 1; + } + + std::string expr; + for (auto i = 1; i < argc; ++i) { + expr += argv[i]; + expr += " "; + } + + auto&& sexp = parse_query(expr, +#ifdef BUILD_PARSE_QUERY_EXPAND + true/*expand*/ +#else + false/*don't expand*/ +#endif + ); + mu_println("{}", sexp.to_string()); + return 0; +} +#endif // BUILD_PARSE_QUERY || BUILD_PARSE_QUERY_EXPAND + + + +#if BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +using TestCase = std::pair<std::string, std::string>; + +static void +test_parser_basic() +{ + std::vector<TestCase> cases = { + // single term + TestCase{R"(a)", R"((_ "a"))"}, + // a and b + TestCase{R"(a and b)", R"((and (_ "a") (_ "b")))"}, + // a and b and c + TestCase{R"(a and b and c)", R"((and (_ "a") (_ "b") (_ "c")))"}, + // a or b + TestCase{R"(a or b)", R"((or (_ "a") (_ "b")))"}, + // a or b and c + TestCase{R"(a or b and c)", R"((or (_ "a") (and (_ "b") (_ "c"))))"}, + // a and b or c + TestCase{R"(a and b or c)", R"((or (and (_ "a") (_ "b")) (_ "c")))"}, + // not a + TestCase{R"(not a)", R"((not (_ "a")))"}, + // lone not + TestCase{R"(not)", R"((_ "not"))"}, + // a and (b or c) + TestCase{R"(a and (b or c))", R"((and (_ "a") (or (_ "b") (_ "c"))))"}, + // not a and not b + TestCase{R"(not a and b)", R"((and (not (_ "a")) (_ "b")))"}, + // a not b + TestCase{R"(a not b)", R"((and (_ "a") (not (_ "b"))))"}, + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first)}; + //mu_message ("'{}' <=> '{}'", sexp.to_string(), test.second); + assert_equal(sexp.to_string(), test.second); + } +} + +static void +test_parser_recover() +{ + std::vector<TestCase> cases = { + // implicit AND + TestCase{R"(a b)", R"((and (_ "a") (_ "b")))"}, + // a or or (second to be used as value) + TestCase{R"(a or and)", R"((or (_ "a") (_ "and")))"}, + // missing end ) + TestCase{R"(a and ()", R"((_ "a"))"}, + // missing end ) + TestCase{R"(a and (b)", R"((and (_ "a") (_ "b")))"}, + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first)}; + assert_equal(sexp.to_string(), test.second); + } +} + + +static void +test_parser_fields() +{ + std::vector<TestCase> cases = { + // simple field + TestCase{R"(s:hello)", R"((subject "hello"))"}, + // field, wildcard, regexp + TestCase{R"(subject:a* recip:/b/)", + R"((and (subject (wildcard "a")) (recip (regex "b"))))"}, + TestCase{R"(from:hello or subject:world)", + R"((or (from "hello") (subject "world")))"}, + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first)}; + assert_equal(sexp.to_string(), test.second); + } +} + +static void +test_parser_expand() +{ + std::vector<TestCase> cases = { + // simple field + TestCase{R"(recip:a)", R"((or (to "a") (cc "a") (bcc "a")))"}, + // field, wildcard, regexp + TestCase{R"(a*)", + R"((or (to (wildcard "a")) (cc (wildcard "a")) (bcc (wildcard "a")) (from (wildcard "a")) (subject (wildcard "a")) (body (wildcard "a")) (embed (wildcard "a"))))"}, + TestCase{R"(a xor contact:b)", + R"((xor (or (to "a") (cc "a") (bcc "a") (from "a") (subject "a") (body "a") (embed "a")) (or (to "b") (cc "b") (bcc "b") (from "b"))))"} + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first, true/*expand*/)}; + assert_equal(sexp.to_string(), test.second); + } +} + + +static void +test_parser_range() +{ + std::vector<TestCase> cases = { + TestCase{R"(size:1)", R"((size (range "1" "1")))"}, + TestCase{R"(size:2..)", R"((size (range "2" "")))"}, + TestCase{R"(size:..1k)", R"((size (range "" "1024")))"}, + TestCase{R"(size:..)", R"((size (range "" "")))"}, + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first, true/*expand*/)}; + assert_equal(sexp.to_string(), test.second); + } +} + +static void +test_parser_optimize() +{ + std::vector<TestCase> cases = { + TestCase{R"(not a)", R"((not (_ "a")))"}, + TestCase{R"(not not a)", R"((_ "a"))"}, + TestCase{R"(not not not a)", R"((not (_ "a")))"}, + TestCase{R"(not not not not a)", R"((_ "a"))"}, + }; + + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first)}; + assert_equal(sexp.to_string(), test.second); + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/query-parser/basic", test_parser_basic); + g_test_add_func("/query-parser/recover", test_parser_recover); + g_test_add_func("/query-parser/fields", test_parser_fields); + g_test_add_func("/query-parser/range", test_parser_range); + g_test_add_func("/query-parser/expand", test_parser_expand); + g_test_add_func("/query-parser/optimize", test_parser_optimize); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query-parser.hh b/lib/mu-query-parser.hh new file mode 100644 index 0000000..72b23a7 --- /dev/null +++ b/lib/mu-query-parser.hh @@ -0,0 +1,114 @@ +/* +** Copyright (C) 2023-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <string> + +#include "mu-xapian-db.hh" + +#include "utils/mu-sexp.hh" +#include "utils/mu-result.hh" +#include "mu-store.hh" + +namespace Mu { +/* + * Some useful symbol-sexps + */ +static inline const auto placeholder_sym = "_"_sym; +static inline const auto phrase_sym = "phrase"_sym; +static inline const auto regex_sym = "regex"_sym; +static inline const auto range_sym = "range"_sym; +static inline const auto wildcard_sym = "wildcard"_sym; + +static inline const auto open_sym = "("_sym; +static inline const auto close_sym = ")"_sym; + +static inline const auto and_sym = "and"_sym; +static inline const auto or_sym = "or"_sym; +static inline const auto xor_sym = "xor"_sym; +static inline const auto not_sym = "not"_sym; +static inline const auto and_not_sym = "and-not"_sym; + + +/* + * We take a query, then parse it into a human-readable s-expression and then + * turn that s-expression into a Xapian query + * + * some query: + * "from:hello or subject:world" + * + * 1. tokenize-query + * => ((from "hello") or (subject "world")) + * + * 2. parse-query + * => (or (from "hello") (subject "world")) + * + * 3. xapian-query + * => Query((Fhello OR Sworld)) + * * + */ + +/** + * Analyze the query expression and express it as a Sexp-list with the sequence + * of elements. + * + * @param expr a search expression + * + * @return Sexp with the sequence of elements + */ +Sexp process_query(const std::string& expr); + +/** + * Parse the query expression and create a parse-tree expressed as an Sexp + * object (tree). + * + * Internally, this processes the stream into element (see process_query()) and + * processes the tokens into a Sexp. This sexp is meant to be human-readable. + * + * @param expr a search expression + * @param expand whether to expand meta-fields (such as '_', 'recip', 'contacts') + * + * @return Sexp with the parse tree + */ +Sexp parse_query(const std::string& expr, bool expand=false); + +/** + * Make a Xapian Query for the given string expression. + * + * This uses parse_query() and turns the S-expression into a Xapian::Query. + * Unlike mere parsing, this uses the information in the store to resolve + * wildcard / regex queries. + * + * @param store the message store + * @param expr a string expression + * @param flavor type of parser to use + * + * @return a Xapian query result or an error. + */ +enum struct ParserFlags { + None = 0 << 0, + SupportNgrams = 1 << 0, /**< Support Xapian's Ngrams for CJK etc. handling */ + XapianParser = 1 << 1, /**< For testing only, use Xapian's + * built-in QueryParser; this is not + * fully compatible with mu, only useful + * for debugging. */ +}; +Result<Xapian::Query> make_xapian_query(const Store& store, const std::string& expr, + ParserFlags flag=ParserFlags::None) noexcept; + +MU_ENABLE_BITOPS(ParserFlags); +} // namespace Mu diff --git a/lib/mu-query-processor.cc b/lib/mu-query-processor.cc new file mode 100644 index 0000000..592beb4 --- /dev/null +++ b/lib/mu-query-processor.cc @@ -0,0 +1,527 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-parser.hh" + +#include <string_view> +#include <variant> +#include <type_traits> +#include <iostream> + +#include "utils/mu-option.hh" +#include <glib.h> +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +/** + * An 'Element' here is a rather rich version of what is traditionally + * considered a (lexical) token. + * + * We try to determine as much as possible during the analysis phase; which is + * quite a bit (given the fairly simple query language), and the parsing phase + * only has to deal with the putting these elements in a tree. + * + * During analysis: + * 1) separate the query into a sequence strings + * 2) for each of these strings + * - Does it look like an Op? ('or', 'and' etc.) --> Op + * - Otherwise: treat as a Basic field ([field]:value) + * - Whitespace in value? -> promote to Phrase + * - otherwise: + * - Is value a regex (in /<regex>/) -> promote to Regex + * - Is value a wildcard (ends in '*') -> promote to Wildcard + * - is value a range (a..b) -> promote to Range + * + * After analysis, we have the sequence of element as a Sexp, which can then be + * fed to the parser. We attempt to make the Sexp as human-readable as possible. + */ +struct Element { + enum struct Bracket { Open, Close} ; + enum struct Op { And, Or, Xor, Not, AndNot }; + + template<typename ValueType> + struct FieldValue { + FieldValue(const ValueType& v): field{}, value{v}{} + + template<typename StringType> + FieldValue(const StringType& fname, const ValueType& v): + field{std::string{fname}}, value{v}{} + template<typename StringType> + FieldValue(const Option<StringType>& fname, const ValueType& v) { + if (fname) + field = std::string{*fname}; + value = v; + } + + Option<std::string> field{}; + ValueType value{}; + }; + struct Basic: public FieldValue<std::string> {using FieldValue::FieldValue;}; + struct Regex: public FieldValue<std::string> {using FieldValue::FieldValue;}; + struct Wildcard: public FieldValue<std::string> {using FieldValue::FieldValue;}; + struct Range: public FieldValue<std::pair<std::string, std::string>> { + using FieldValue::FieldValue; }; + + using ValueType = std::variant< + /* */ + Bracket, + /* op */ + Op, + /* string values */ + std::string, + /* value types */ + Basic, + Regex, + Wildcard, + Range + >; + + // helper + template <typename T, typename U> + struct decay_equiv: + std::is_same<typename std::decay<T>::type, U>::type {}; + + Element(Bracket b): value{b} {} + Element(Op op): value{op} {} + + template<typename T, + typename std::enable_if<std::is_base_of<class FieldValue<T>, T>::value>::type = 0> + Element(const std::string& field, const T& val): value{T{field, val}} {} + + Element(const std::string& val): value{val} {} + + template<typename T> + Option<T&> get_opt() { + if (std::holds_alternative<T>(value)) + return std::get<T>(value); + else + return Nothing; + } + + Sexp sexp() const { + return std::visit([](auto&& arg)->Sexp { + + auto field_sym = [](const Option<std::string>& field) { + return field ? Sexp::Symbol{*field} : placeholder_sym; + }; + + using T = std::decay_t<decltype(arg)>; + + if constexpr (std::is_same_v<T, Bracket>) { + switch(arg) { + case Bracket::Open: + return open_sym; + case Bracket::Close: + return close_sym; + default: + throw std::logic_error("invalid bracket type"); + } + } else if constexpr (std::is_same_v<T, Op>) { + switch(arg) { + case Op::And: + return and_sym; + case Op::Or: + return or_sym; + case Op::Xor: + return xor_sym; + case Op::Not: + return not_sym; + case Op::AndNot: + return and_not_sym; + default: + throw std::logic_error("invalid op type"); + } + } else if constexpr (std::is_same_v<T, Basic>) { + return Sexp { field_sym(arg.field), arg.value }; + } else if constexpr (std::is_same_v<T, Regex>) { + return Sexp { field_sym(arg.field), Sexp{ regex_sym, arg.value}}; + } else if constexpr (std::is_same_v<T, Wildcard>) { + return Sexp { field_sym(arg.field), Sexp{ wildcard_sym, arg.value}}; + } else if constexpr (std::is_same_v<T, Range>) { + return Sexp {field_sym(arg.field), + Sexp{ range_sym, arg.value.first, arg.value.second }}; + } else if constexpr (std::is_same_v<T, std::string>) { + throw std::logic_error("no bare strings should be here"); + } else + throw std::logic_error("uninvited visitor"); + }, value); + } + + ValueType value; +}; + +using Elements = std::vector<Element>; + + + +/** + * Remove first character from string and return it. + * + * @param[in,out] str a string + * @param[in,out] pos position in _original_ string + * + * @return a char or 0 if there is none. + */ +static char +read_char(std::string& str, size_t& pos) +{ + if (str.empty()) + return {}; + + auto kar{str.at(0)}; + str.erase(0, 1); + ++pos; + + return kar; +} + +/** + * Restore kar at the beginning of the string + * + * @param[in,out] str a string + * @param[in,out] pos position in _original_ string + * @param kar a character + */ +static void +unread_char(std::string& str, size_t& pos, char kar) +{ + str = kar + str; + --pos; +} + + +/** + * Remove the the next element from the string and return it + * + * @param[in,out] str a string + * @param[in,out] pos position in _original_ string * + * + * @return an Element or Nothing + */ +static Option<Element> +next_element(std::string& str, size_t& pos) +{ + bool quoted{}, escaped{}; + std::string value{}; + + auto is_separator = [](char c) { return c == ' '|| c == '(' || c == ')'; }; + + while (!str.empty()) { + + auto kar = read_char(str, pos); + + if (kar == '\\') { + escaped = !escaped; + if (escaped) + continue; + } + + if (kar == '"' && !escaped) { + if (!escaped && quoted) + return Element{value}; + else { + quoted = true; + continue; + } + } + + if (!quoted && !escaped && is_separator(kar)) { + if (!value.empty()) { + unread_char(str, pos, kar); + return Element{value}; + } + + if (quoted || kar == ' ') + continue; + + switch (kar) { + case '(': + return Element{Element::Bracket::Open}; + case ')': + return Element{Element::Bracket::Close}; + default: + break; + } + } + + value += kar; + escaped = false; + } + + if (value.empty()) + return Nothing; + else + return Element{value}; +} + + +static Option<Element> +opify(Element&& element) +{ + auto&& str{element.get_opt<std::string>()}; + if (!str) + return element; + + static const std::unordered_map<std::string, Element::Op> ops = { + { "and", Element::Op::And }, + { "or", Element::Op::Or}, + { "xor", Element::Op::Xor }, + { "not", Element::Op::Not }, + // AndNot only appears during parsing. + }; + + if (auto&& it = ops.find(utf8_flatten(*str)); it != ops.end()) + element.value = it->second; + + return element; +} + +static Option<Element> +basify(Element&& element) +{ + auto&& str{element.get_opt<std::string>()}; + if (!str) + return element; + + const auto pos = str->find(':'); + if (pos == std::string::npos) { + element.value = Element::Basic{*str}; + return element; + } + + const auto fname{str->substr(0, pos)}; + if (auto&& field{field_from_name(fname)}; field) { + auto val{str->substr(pos + 1)}; + if (field == Field::Id::Flags) { + if (auto&& finfo{flag_info(val)}; finfo) + element.value = Element::Basic{field->name, + std::string{finfo->name}}; + else + element.value = Element::Basic{*str}; + } else if (field == Field::Id::Priority) { + if (auto&& prio{priority_from_name(val)}; prio) + element.value = Element::Basic{field->name, + std::string{priority_name(*prio)}}; + else + element.value = Element::Basic{*str}; + } else + element.value = Element::Basic{std::string{field->name}, + str->substr(pos + 1)}; + } else if (field_is_combi(fname)) + element.value = Element::Basic{fname, str->substr(pos +1)}; + else + element.value = Element::Basic{*str}; + + return element; +} + +static Option<Element> +wildcardify(Element&& element) +{ + auto&& basic{element.get_opt<Element::Basic>()}; + if (!basic) + return element; + + auto&& val{basic->value}; + if (val.size() < 2 || val[val.size()-1] != '*') + return element; + + val.erase(val.size() - 1); + element.value = Element::Wildcard{basic->field, val}; + + return element; +} + +static Option<Element> +regexpify(Element&& element) +{ + auto&& str{element.get_opt<Element::Basic>()}; + if (!str) + return element; + + auto&& val{str->value}; + if (val.size() < 3 || val[0] != '/' || val[val.size()-1] != '/') + return element; + + val.erase(val.size() - 1); + val.erase(0, 1); + element.value = Element::Regex{str->field, std::move(val)}; + + return element; +} + +// handle range-fields: Size, Date, Changed +static Option<Element> +rangify(Element&& element) +{ + auto&& str{element.get_opt<Element::Basic>()}; + if (!str) + return element; + + if (!str->field) + return element; + + auto&& field = field_from_name(*str->field); + if (!field || !field->is_range()) + return element; + + /* yes: get the range */ + auto&& range = std::invoke([&]()->std::pair<std::string, std::string> { + const auto val{str->value}; + const auto pos{val.find("..")}; + + if (pos == std::string::npos) + return { val, val }; + else + return {val.substr(0, pos), val.substr(pos + 2)}; + }); + + if (field->id == Field::Id::Size) { + int64_t s1{range.first.empty() ? -1 : + parse_size(range.first, false/*first*/).value_or(-1)}; + int64_t s2{range.second.empty() ? -1 : + parse_size(range.second, true/*last*/).value_or(-1)}; + if (s2 >= 0 && s1 > s2) + std::swap(s1, s2); + element.value = Element::Range{str->field, + {s1 < 0 ? "" : std::to_string(s1), + s2 < 0 ? "" : std::to_string(s2)}}; + + } else if (field->id == Field::Id::Date || field->id == Field::Id::Changed) { + auto tstamp=[](auto&& str, auto&& first)->int64_t { + return str.empty() ? -1 : + parse_date_time(str, first ,false/*local*/).value_or(-1); + }; + int64_t lower{tstamp(range.first, true/*lower*/)}; + int64_t upper{tstamp(range.second, false/*upper*/)}; + if (lower >= 0 && upper >= 0 && lower > upper) { + // can't simply swap due to rounding up/down + lower = tstamp(range.second, true/*lower*/); + upper = tstamp(range.first, false/*upper*/); + } + // use "Zulu" time. + element.value = Element::Range{ + str->field, + {lower < 0 ? "" : + mu_format("{:%FT%TZ}",mu_time(lower, true/*utc*/)), + upper < 0 ? "" : + mu_format("{:%FT%TZ}", mu_time(upper, true/*utc*/))}}; + } + + return element; +} + +static Elements +process(const std::string& expr) +{ + Elements elements{}; + size_t offset{0}; + + /* all control chars become SPC */ + std::string str{expr}; + for (auto& c: str) + c = ::iscntrl(c) ? ' ' : c; + + while(!str.empty()) { + auto&& element = next_element(str, offset) + .and_then(opify) + .and_then(basify) + .and_then(regexpify) + .and_then(wildcardify) + .and_then(rangify); + if (element) + elements.emplace_back(std::move(element.value())); + } + + return elements; +} + +Sexp +Mu::process_query(const std::string& expr) +{ + const auto& elements{::process(expr)}; + + Sexp sexp{}; + for (auto&& elm: elements) + sexp.add(elm.sexp()); + + return sexp; +} + +#ifdef BUILD_PROCESS_QUERY +int +main (int argc, char *argv[]) +{ + if (argc < 2) { + mu_printerrln("expected: process-query <query>"); + return 1; + } + + std::string expr; + for (auto i = 1; i < argc; ++i) { + expr += argv[i]; + expr += " "; + } + + auto sexp = process_query(expr); + mu_println("{}", sexp.to_string()); + + return 0; +} +#endif /*BUILD_ANALYZE_QUERY*/ + +#if BUILD_TESTS +/* + * + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +using TestCase = std::pair<std::string, std::string>; + +static void +test_processor() +{ + std::vector<TestCase> cases = { + // basics + TestCase{R"(hello world)", R"(((_ "hello") (_ "world")))"}, + TestCase{R"(maildir:/"hello world")", R"(((maildir "/hello world")))"}, + TestCase{R"(flag:deleted)", R"(((_ "flag:deleted")))"} // non-existing flags + }; + + for (auto&& test: cases) { + auto&& sexp{process_query(test.first)}; + assert_equal(sexp.to_string(), test.second); + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/query-parser/processor", test_processor); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query-results.hh b/lib/mu-query-results.hh new file mode 100644 index 0000000..0123ab4 --- /dev/null +++ b/lib/mu-query-results.hh @@ -0,0 +1,422 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_QUERY_RESULTS_HH__ +#define MU_QUERY_RESULTS_HH__ + +#include <algorithm> +#include <limits> +#include <stdexcept> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <limits> +#include <ostream> +#include <cmath> +#include <memory> + +#include <unistd.h> +#include <fcntl.h> +#include <glib.h> + +#include <mu-xapian-db.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> + +#include <message/mu-message.hh> + +namespace Mu { + +/** + * This implements a QueryResults structure, which capture the results of a + * Xapian query, and a QueryResultsIterator, which gives C++-compliant iterator + * to go over the results. and finally QueryThreader (in query-threader.cc) which + * calculates the threads, using the JWZ algorithm. + */ + +/// Flags that influence now matches are presented (or skipped) +enum struct QueryFlags { + None = 0, /**< no flags */ + Descending = 1 << 0, /**< sort z->a */ + SkipUnreadable = 1 << 1, /**< skip unreadable msgs */ + SkipDuplicates = 1 << 2, /**< skip duplicate msgs */ + IncludeRelated = 1 << 3, /**< include related msgs */ + Threading = 1 << 4, /**< calculate threading info */ + // internal + Leader = 1 << 5, /**< This is the leader query (for internal use + * only)*/ +}; +MU_ENABLE_BITOPS(QueryFlags); + +/// Stores all the essential information for sorting the results. +struct QueryMatch { + /// Flags for a match (message) found + enum struct Flags { + None = 0, /**< No Flags */ + Leader = 1 << 0, /**< Mark direct matches as leader */ + Related = 1 << 1, /**< A related message */ + Unreadable = 1 << 2, /**< No readable file */ + Duplicate = 1 << 3, /**< Message-id seen before */ + + Root = 1 << 10, /**< Is this the thread-root? */ + First = 1 << 11, /**< Is this the first message in a thread? */ + Last = 1 << 12, /**< Is this the last message in a thread? */ + Orphan = 1 << 13, /**< Is this message without a parent? */ + HasChild = 1 << 14, /**< Does this message have a child? */ + + ThreadSubject = 1 << 20, /**< Message holds subject for (sub)thread */ + }; + + Flags flags{Flags::None}; /**< Flags */ + std::string date_key; /**< The date-key (for sorting all sub-root levels) */ + // the thread subject is the subject of the first message in a thread, + // and any message that has a different subject compared to its predecessor + // (ignoring prefixes such as Re:) + // + // otherwise, it is empty. + std::string subject; /**< subject for this message */ + size_t thread_level{}; /**< The thread level */ + std::string thread_path; /**< The hex-numerial path in the thread, ie. '00:01:0a' */ + std::string thread_date; /**< date of newest message in thread */ + + bool operator<(const QueryMatch& rhs) const { return date_key < rhs.date_key; } + + bool has_flag(Flags flag) const; +}; + +MU_ENABLE_BITOPS(QueryMatch::Flags); + +inline bool +QueryMatch::has_flag(QueryMatch::Flags flag) const +{ + return any_of(flags & flag); +} + +/* LCOV_EXCL_START */ +static inline std::ostream& +operator<<(std::ostream& os, QueryMatch::Flags mflags) +{ + if (mflags == QueryMatch::Flags::None) { + os << "<none>"; + return os; + } + + if (any_of(mflags & QueryMatch::Flags::Leader)) + os << "leader "; + if (any_of(mflags & QueryMatch::Flags::Unreadable)) + os << "unreadable "; + if (any_of(mflags & QueryMatch::Flags::Duplicate)) + os << "dup "; + + if (any_of(mflags & QueryMatch::Flags::Root)) + os << "root "; + if (any_of(mflags & QueryMatch::Flags::Related)) + os << "related "; + if (any_of(mflags & QueryMatch::Flags::First)) + os << "first "; + if (any_of(mflags & QueryMatch::Flags::Last)) + os << "last "; + if (any_of(mflags & QueryMatch::Flags::Orphan)) + os << "orphan "; + if (any_of(mflags & QueryMatch::Flags::HasChild)) + os << "has-child "; + + return os; +} + +inline std::ostream& +operator<<(std::ostream& os, const QueryMatch& qmatch) +{ + os << "qm:[" << qmatch.thread_path << "]: " // " (" << qmatch.thread_level << "): " + << "> date:<" << qmatch.date_key << "> " + << "flags:{" << qmatch.flags << "}"; + + return os; +} +/* LCOV_EXCL_STOP*/ + +using QueryMatches = std::unordered_map<Xapian::docid, QueryMatch>; + + +/// +/// This is a view over the Xapian::MSet, which can optionally filter unreadable +/// / duplicate messages. +/// +/// Note, we internally skip unreadable/duplicate messages (when asked too); those +/// skipped ones do _not_ count towards the max_size +/// +class QueryResultsIterator { +public: + using iterator_category = std::output_iterator_tag; + using value_type = Message; + using difference_type = void; + using pointer = void; + using reference = void; + + QueryResultsIterator(Xapian::MSetIterator mset_it, QueryMatches& query_matches) + : mset_it_{mset_it}, query_matches_{query_matches} { + } + + /** + * Increment the iterator (we don't support post-increment) + * + * @return an updated iterator, or end() if we were already at end() + */ + QueryResultsIterator& operator++() { + ++mset_it_; + mdoc_ = Nothing; + return *this; + } + + /** + * (Non)Equivalence operators + * + * @param rhs some other iterator + * + * @return true or false + */ + bool operator==(const QueryResultsIterator& rhs) const { return mset_it_ == rhs.mset_it_; } + bool operator!=(const QueryResultsIterator& rhs) const { return mset_it_ != rhs.mset_it_; } + + QueryResultsIterator& operator*() { return *this; } + const QueryResultsIterator& operator*() const { return *this; } + + + /** + * Get the Xapian::Document this iterator is pointing at, + * or an empty document when looking at end(). + * + * @return a document + */ + Option<Xapian::Document> document() const { + return xapian_try([this]()->Option<Xapian::Document> { + auto doc{mset_it_.get_document()}; + if (doc.get_docid() == 0) + return Nothing; + else + return Some(std::move(doc)); + }, Nothing); + } + + + /** + * get the corresponding Message for this iter, if any + * + * @return a Message or Nothing + */ + Option<Message> message() const { + if (auto&& xdoc{document()}; !xdoc) + return Nothing; + else if (auto&& doc{Message::make_from_document(std::move(xdoc.value()))}; + !doc) + return Nothing; + else + return Some(std::move(doc.value())); + } + + /** + * Get the doc-id for the document this iterator is pointing at, or 0 + * when looking at end. + * + * @return a doc-id. + */ + Xapian::docid doc_id() const { return *mset_it_; } + + /** + * Get the message-id for the document (message) this iterator is + * pointing at, or not when not available + * + * @return a message-id + */ + Option<std::string> message_id() const noexcept { + return opt_string(Field::Id::MessageId); + } + + /** + * Get the thread-id for the document (message) this iterator is + * pointing at, or Nothing. + * + * @return a message-id + */ + Option<std::string> thread_id() const noexcept { + return opt_string(Field::Id::ThreadId); + } + + /** + * Get the file-system path for the document (message) this iterator is + * pointing at, or Nothing. + * + * @return a filesystem path + */ + Option<std::string> path() const noexcept { + return opt_string(Field::Id::Path); + } + + /** + * Get the a sortable date str for the document (message) the iterator + * is pointing at. pointing at, or Nothing. This (encoded) string + * has the same sort-order as the corresponding date. + * + * @return a filesystem path + */ + Option<std::string> date_str() const noexcept { + return opt_string(Field::Id::Date); + } + + /** + * Get the subject for the document (message) this iterator is pointing + * at. + * + * @return the subject + */ + Option<std::string> subject() const noexcept { + return opt_string(Field::Id::Subject); + } + + /** + * Get the references for the document (messages) this is iterator is + * pointing at, or empty if pointing at end of if no references are + * available. + * + * @return references + */ + std::vector<std::string> references() const noexcept { + return mu_document().string_vec_value(Field::Id::References); + } + + /** + * Get some value from the document, or Nothing if empty. + * + * @param id a message field id + * + * @return the value + */ + Option<std::string> opt_string(Field::Id id) const noexcept { + if (auto&& val{mu_document().string_value(id)}; val.empty()) + return Nothing; + else + return Some(std::move(val)); + } + + /** + * Get the Query match info for this message. + * + * @return the match info. + */ + QueryMatch& query_match() { + g_assert(query_matches_.find(doc_id()) != query_matches_.end()); + return query_matches_.find(doc_id())->second; + } + const QueryMatch& query_match() const { + g_assert(query_matches_.find(doc_id()) != query_matches_.end()); + return query_matches_.find(doc_id())->second; + } + +private: + /** + * Get a (cached) reference for the Mu::Document corresponding + * to the current iter. + * + * @return cached mu document, + */ + const Mu::Document& mu_document() const { + if (!mdoc_) { + if (auto xdoc = document(); !xdoc) + std::runtime_error("iter without document"); + else + mdoc_ = Mu::Document{xdoc.value()}; + } + return mdoc_.value(); + } + + mutable Option<Mu::Document> mdoc_; // cache. + Xapian::MSetIterator mset_it_; + QueryMatches& query_matches_; +}; + + +static inline auto +format_as(const QueryResultsIterator& it) +{ + return it.path().value_or("<no path>"); +} + +constexpr auto MaxQueryResultsSize = std::numeric_limits<size_t>::max(); + +class QueryResults { +public: + /// Helper types + using iterator = QueryResultsIterator; + using const_iterator = const iterator; + + /** + * Construct a QueryResults object + * + * @param mset an Xapian::MSet with matches + */ + QueryResults(const Xapian::MSet& mset, QueryMatches&& query_matches) + : mset_{mset}, query_matches_{std::move(query_matches)} + { + } + /** + * Is this QueryResults object empty (ie., no matches)? + * + * @return true are false + */ + bool empty() const { return mset_.empty(); } + + /** + * Get the number of matches in this QueryResult + * + * @return number of matches + */ + size_t size() const { return mset_.size(); } + + /** + * Get the begin iterator to the results. + * + * @return iterator + */ + const iterator begin() const { return QueryResultsIterator(mset_.begin(), query_matches_); } + + /** + * Get the end iterator to the results. + * + * @return iterator + */ + const_iterator end() const { return QueryResultsIterator(mset_.end(), query_matches_); } + + /** + * Get the query-matches for these QueryResults. The non-const + * version can be use to _steal_ the query results, by moving + * them. + * + * @return query-matches + */ + const QueryMatches& query_matches() const { return query_matches_; } + QueryMatches& query_matches() { return query_matches_; } + +private: + const Xapian::MSet mset_; + mutable QueryMatches query_matches_; +}; + +} // namespace Mu + +#endif /* MU_QUERY_RESULTS_HH__ */ diff --git a/lib/mu-query-threads.cc b/lib/mu-query-threads.cc new file mode 100644 index 0000000..6d99281 --- /dev/null +++ b/lib/mu-query-threads.cc @@ -0,0 +1,957 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-threads.hh" +#include <message/mu-message.hh> + +#include <set> +#include <unordered_set> +#include <list> +#include <cassert> +#include <cstring> +#include <iostream> +#include <iomanip> + +#include <utils/mu-option.hh> + +using namespace Mu; + +struct Container { + using Containers = std::vector<Container*>; + + Container() = default; + Container(Option<QueryMatch&> msg) : query_match{msg} {} + Container(const Container&) = delete; + Container(Container&&) = default; + + void add_child(Container& new_child) + { + new_child.parent = this; + children.emplace_back(&new_child); + } + void remove_child(Container& child) + { + children.erase(find_child(child)); + assert(!has_child(child)); + } + + Containers::iterator find_child(Container& child) + { + return std::find_if(children.begin(), children.end(), [&](auto&& c) { + return c == &child; + }); + } + Containers::const_iterator find_child(Container& child) const + { + return std::find_if(children.begin(), children.end(), [&](auto&& c) { + return c == &child; + }); + } + bool has_child(Container& child) const { return find_child(child) != children.cend(); } + + bool is_reachable(Container* other) const + { + auto up{ur_parent()}; + return up && up == other->ur_parent(); + } + template <typename Func> void for_each_child(Func&& func) + { + auto it{children.rbegin()}; + while (it != children.rend()) { + auto next = std::next(it); + func(*it); + it = next; + } + } + // During sorting, this is the cached value for the (recursive) date-key + // of this container -- ie.. either the one from the first of its + // children, or from its query-match, if it has no children. + // + // Note that the sub-root-levels of threads are always sorted by date, + // in ascending order, regardless of whatever sorting was specified for + // the root-level. + + std::string thread_date_key; + + Option<QueryMatch&> query_match; + bool is_nuked{}; + Container* parent{}; + Containers children; + + using ContainerVec = std::vector<Container*>; + + private: + const Container* ur_parent() const + { + assert(this->parent != this); + return parent ? parent->ur_parent() : this; + } +}; + +using Containers = Container::Containers; +using ContainerVec = Container::ContainerVec; + +/* LCOV_EXCL_START */ +static std::ostream& +operator<<(std::ostream& os, const Container& container) +{ + os << "container: " << std::right << std::setw(10) << &container + << ": parent: " << std::right << std::setw(10) << container.parent << " [" + << container.thread_date_key << "]" + << "\n children: "; + + for (auto&& c : container.children) + os << std::right << std::setw(10) << c << " "; + + os << (container.is_nuked ? " nuked" : ""); + + if (container.query_match) + os << "\n " << container.query_match.value(); + + return os; +} +/* LCOV_EXCL_STOP */ + +using IdTable = std::unordered_map<std::string, Container>; +using DupTable = std::multimap<std::string, Container>; + +static void +handle_duplicates(IdTable& id_table, DupTable& dup_table) +{ + size_t n{}; + + for (auto&& dup : dup_table) { + const auto msgid{dup.first}; + auto it = id_table.find(msgid); + if (it == id_table.end()) + continue; + + // add duplicates as fake children + char buf[32]; + ::snprintf(buf, sizeof(buf), "dup-%zu", ++n); + it->second.add_child(id_table.emplace(buf, std::move(dup.second)).first->second); + } +} + +template <typename QueryResultsType> +static IdTable +determine_id_table(QueryResultsType& qres) +{ + // 1. For each query_match + IdTable id_table; + DupTable dups; + for (auto&& mi : qres) { + const auto msgid{mi.message_id().value_or(*mi.path())}; + // Step 0 (non-JWZ): filter out dups, handle those at the end + if (mi.query_match().has_flag(QueryMatch::Flags::Duplicate)) { + dups.emplace(msgid, mi.query_match()); + continue; + } + // 1.A If id_table contains an empty Container for this ID: + // Store this query_match (query_match) in the Container's query_match (value) slot. + // Else: + // Create a new Container object holding this query_match (query-match); + // Index the Container by Query_Match-ID + auto c_it = id_table.find(msgid); + auto& container = [&]() -> Container& { + if (c_it != id_table.end()) { + if (!c_it->second.query_match) // hmm, dup? + c_it->second.query_match = mi.query_match(); + return c_it->second; + } else { + // Else: + // Create a new Container object holding this query_match + // (query-match); Index the Container by Query_Match-ID + return id_table.emplace(msgid, mi.query_match()).first->second; + } + }(); + + // We sort by date (ascending), *except* for the root; we don't + // know what query_matchs will be at the root level yet, so remember + // both. Moreover, even when sorting the top-level in descending + // order, still sort the thread levels below that in ascending + // order. + container.thread_date_key = container.query_match->date_key = + mi.date_str().value_or(""); + // initial guess for the thread-date; might be updated + // later. + + // remember the subject, we use it to determine the (sub)thread subject + container.query_match->subject = mi.subject().value_or(""); + + // 1.B + // For each element in the query_match's References field: + Container* parent_ref_container{}; + for (const auto& ref : mi.references()) { + // grand_<n>-parent -> grand_<n-1>-parent -> ... -> parent. + + // Find a Container object for the given Query_Match-ID; If it exists, use + // it; otherwise make one with a null Query_Match. + auto ref_container = [&]() -> Container* { + auto ref_it = id_table.find(ref); + if (ref_it == id_table.end()) + ref_it = id_table.emplace(ref, Nothing).first; + return &ref_it->second; + }(); + + // Link the References field's Containers together in the order implied + // by the References header. + // * If they are already linked, don't change the existing links. + // + // * Do not add a link if adding that link would introduce a loop: that is, + // before asserting A->B, search down the children of B to see if A is + // reachable, and also search down the children of A to see if B is + // reachable. If either is already reachable as a child of the other, + // don't add the link. + if (parent_ref_container && !ref_container->parent) { + if (!parent_ref_container->is_reachable(ref_container)) + parent_ref_container->add_child(*ref_container); + // else + // g_message ("%u: reachable %s -> %s", __LINE__, + // msgid.c_str(), ref.c_str()); + } + + parent_ref_container = ref_container; + } + + // Add the query_match to the chain. + if (parent_ref_container && !container.parent) { + if (!parent_ref_container->is_reachable(&container)) + parent_ref_container->add_child(container); + // else + // g_message ("%u: reachable %s -> parent", __LINE__, + // msgid.c_str()); + } + } + + // non-JWZ: add duplicate messages. + handle_duplicates(id_table, dups); + + return id_table; +} + +/// Recursively walk all containers under the root set. +/// For each container: +/// +/// If it is an empty container with no children, nuke it. +/// +/// Note: Normally such containers won't occur, but they can show up when two +/// query_matchs have References lines that disagree. For example, assuming A and +/// B are query_matchs, and 1, 2, and 3 are references for query_matchs we haven't +/// seen: +/// +/// A has references: 1, 2, 3 +/// B has references: 1, 3 +/// +/// There is ambiguity as to whether 3 is a child of 1 or of 2. So, +/// depending on the processing order, we might end up with either +/// +/// -- 1 +/// |-- 2 +/// \-- 3 +/// |-- A +/// \-- B +/// +/// or +/// +/// -- 1 +/// |-- 2 <--- non root childless container! +/// \-- 3 +/// |-- A +/// \-- B +/// +/// If the Container has no Query_Match, but does have children, remove this +/// container but promote its children to this level (that is, splice them in +/// to the current child list.) +/// +/// Do not promote the children if doing so would promote them to the root +/// set -- unless there is only one child, in which case, do. + +static void +prune(Container* child) +{ + Container* container{child->parent}; + + for (auto& grandchild : child->children) { + grandchild->parent = container; + if (container) + container->children.emplace_back(grandchild); + } + + child->children.clear(); + child->is_nuked = true; + + if (container) + container->remove_child(*child); +} + +static bool +prune_empty_containers(Container& container) +{ + Containers to_prune; + + container.for_each_child([&](auto& child) { + if (prune_empty_containers(*child)) + to_prune.emplace_back(child); + }); + + for (auto& child : to_prune) + prune(child); + + // Never nuke these. + if (container.query_match) + return false; + + // If it is an empty container with no children, nuke it. + // + // If the Container is empty, but does have children, remove this + // container but promote its children to this level (that is, splice them in + // to the current child list.) + // + // Do not promote the children if doing so would promote them to the root + // set -- unless there is only one child, in which case, do. + // const auto rootset_child{!container.parent->parent}; + if (container.parent || container.children.size() <= 1) + return true; // splice/nuke it. + + return false; +} + +static void +prune_empty_containers(IdTable& id_table) +{ + for (auto&& item : id_table) { + auto& child(item.second); + if (child.parent) + continue; // not a root child. + + if (prune_empty_containers(item.second)) + prune(&child); + } +} + +// +// Sorting. +// + +/// Register some information about a match (i.e., message) that we can use for +/// subsequent queries. +using ThreadPath = std::vector<unsigned>; +inline std::string +to_string(const ThreadPath& tpath, size_t digits) +{ + std::string str; + str.reserve(tpath.size() * digits); + + bool first{true}; + for (auto&& segm : tpath) { + str += mu_format("{}{:0{}x}", first ? "" : ":", segm, digits); + first = false; + } + + return str; +} + +static bool // compare subjects, ignore anything before the last ':<space>*' +subject_matches(const std::string& sub1, const std::string& sub2) +{ + auto search_str = [](const std::string& s) -> const char* { + const auto pos = s.find_last_of(':'); + if (pos == std::string::npos) + return s.c_str(); + else { + const auto pos2 = s.find_first_not_of(' ', pos + 1); + return s.c_str() + (pos2 == std::string::npos ? pos : pos2); + } + }; + + return g_strcmp0(search_str(sub1), search_str(sub2)) == 0; +} + +static bool +update_container(Container& container, + bool descending, + ThreadPath& tpath, + size_t seg_size, + const std::string& prev_subject = "") +{ + if (!container.children.empty()) { + Container* first = container.children.front(); + if (first->query_match) + first->query_match->flags |= QueryMatch::Flags::First; + Container* last = container.children.back(); + if (last->query_match) + last->query_match->flags |= QueryMatch::Flags::Last; + } + + if (!container.query_match) + return false; // nothing else to do. + + auto& qmatch(*container.query_match); + if (!container.parent) + qmatch.flags |= QueryMatch::Flags::Root; + else if (!container.parent->query_match) + qmatch.flags |= QueryMatch::Flags::Orphan; + + if (!container.children.empty()) + qmatch.flags |= QueryMatch::Flags::HasChild; + + if (qmatch.has_flag(QueryMatch::Flags::Root) || prev_subject.empty() || + !subject_matches(prev_subject, qmatch.subject)) + qmatch.flags |= QueryMatch::Flags::ThreadSubject; + + if (descending && container.parent) { + // trick xapian by giving it "inverse" sorting key so our + // ascending-date sorted threads stay in that order + tpath.back() = ((1U << (4 * seg_size)) - 1) - tpath.back(); + } + + qmatch.thread_path = to_string(tpath, seg_size); + qmatch.thread_level = tpath.size() - 1; + + // ensure thread root comes before its children + if (descending) + qmatch.thread_path += ":z"; + + return true; +} + +static void +update_containers(Containers& children, + bool descending, + ThreadPath& tpath, + size_t seg_size, + std::string& prev_subject) +{ + size_t idx{0}; + + for (auto&& c : children) { + tpath.emplace_back(idx++); + if (c->query_match) { + update_container(*c, descending, tpath, seg_size, prev_subject); + prev_subject = c->query_match->subject; + } + update_containers(c->children, descending, tpath, seg_size, prev_subject); + tpath.pop_back(); + } +} + +static void +update_containers(ContainerVec& root_vec, bool descending, size_t n) +{ + ThreadPath tpath; + tpath.reserve(n); + + const auto seg_size = static_cast<size_t>(std::ceil(std::log2(n) / 4.0)); + /*note: 4 == std::log2(16)*/ + + size_t idx{0}; + for (auto&& c : root_vec) { + tpath.emplace_back(idx++); + std::string prev_subject; + if (update_container(*c, descending, tpath, seg_size)) + prev_subject = c->query_match->subject; + update_containers(c->children, descending, tpath, seg_size, prev_subject); + tpath.pop_back(); + } +} + +static void +sort_container(Container& container) +{ + // 1. childless container. + if (container.children.empty()) + return; // no children; nothing to sort. + + // 2. container with children. + // recurse, depth-first: sort the children + for (auto& child : container.children) + sort_container(*child); + + // now sort this level. + std::sort(container.children.begin(), container.children.end(), [&](auto&& c1, auto&& c2) { + return c1->thread_date_key < c2->thread_date_key; + }); + + // and 'bubble up' the date of the *newest* message with a date. We + // reasonably assume that it's later than its parent. + const auto& newest_date = container.children.back()->thread_date_key; + if (!newest_date.empty()) + container.thread_date_key = newest_date; +} + +static void +sort_siblings(IdTable& id_table, bool descending) +{ + if (id_table.empty()) + return; + + // unsorted vec of root containers. We can + // only sort these _after_ sorting the children. + ContainerVec root_vec; + for (auto&& item : id_table) { + if (!item.second.parent && !item.second.is_nuked) + root_vec.emplace_back(&item.second); + } + + // now sort all threads _under_ the root set (by date/ascending) + for (auto&& c : root_vec) + sort_container(*c); + + // and then sort the root set. + // + // The difference with the sub-root containers is that at the top-level, + // we can sort either in ascending or descending order, while on the + // subroot level it's always in ascending order. + // + // Note that unless we're testing, _xapian_ will handle + // the ascending/descending of the top level. + std::sort(root_vec.begin(), root_vec.end(), [&](auto&& c1, auto&& c2) { +#ifdef BUILD_TESTS + if (descending) + return c2->thread_date_key < c1->thread_date_key; + else +#endif /*BUILD_TESTS*/ + return c1->thread_date_key < c2->thread_date_key; + }); + + // now all is sorted... final step is to determine thread paths and + // other flags. + update_containers(root_vec, descending, id_table.size()); +} + +/* LCOV_EXCL_START */ +static std::ostream& +operator<<(std::ostream& os, const IdTable& id_table) +{ + os << "------------------------------------------------\n"; + for (auto&& item : id_table) { + os << item.first << " => " << item.second << "\n"; + } + os << "------------------------------------------------\n"; + + std::set<std::string> ids; + for (auto&& item : id_table) { + if (item.second.query_match) + ids.emplace(item.second.query_match->thread_path); + } + + for (auto&& id : ids) { + auto it = std::find_if(id_table.begin(), id_table.end(), [&](auto&& item) { + return item.second.query_match && + item.second.query_match->thread_path == id; + }); + assert(it != id_table.end()); + os << it->first << ": " << it->second << '\n'; + } + return os; +} +/* LCOV_EXCL_STOP */ + +template <typename Results> +static void +calculate_threads_real(Results& qres, bool descending) +{ + // Step 1: build the id_table + auto id_table{determine_id_table(qres)}; + + if (g_test_verbose()) + std::cout << "*** id-table(1):\n" << id_table << "\n"; + + // // Step 2: get the root set + // // Step 3: discard id_table + // Nope: id-table owns the containers. + // Step 4: prune empty containers + prune_empty_containers(id_table); + + // Step 5: group root-set by subject. + // Not implemented. + + // Step 6: we're done threading + + // Step 7: sort siblings. The segment-size is the number of hex-digits + // in the thread-path string (so we can lexically compare them.) + sort_siblings(id_table, descending); + + // Step 7a:. update querymatches + for (auto&& item : id_table) { + Container& c{item.second}; + if (c.query_match) + c.query_match->thread_date = c.thread_date_key; + } + // if (g_test_verbose()) + // std::cout << "*** id-table(2):\n" << id_table << "\n"; +} + +void +Mu::calculate_threads(Mu::QueryResults& qres, bool descending) +{ + calculate_threads_real(qres, descending); +} + +#ifdef BUILD_TESTS + +struct MockQueryResult { + MockQueryResult(const std::string& message_id_arg, + const std::string& date_arg, + const std::vector<std::string>& refs_arg = {}) + : message_id_{message_id_arg}, date_{date_arg}, refs_{refs_arg} + { + } + MockQueryResult(const std::string& message_id_arg, + const std::vector<std::string>& refs_arg = {}) + : MockQueryResult(message_id_arg, "", refs_arg) + { + } + Option<std::string> message_id() const { return message_id_; } + Option<std::string> path() const { return path_; } + Option<std::string> date_str() const { return date_; } + Option<std::string> subject() const { return subject_; } + QueryMatch& query_match() { return query_match_; } + const QueryMatch& query_match() const { return query_match_; } + const std::vector<std::string>& references() const { return refs_; } + + std::string path_; + std::string message_id_; + QueryMatch query_match_{}; + std::string date_; + std::string subject_; + std::vector<std::string> refs_; +}; + +using MockQueryResults = std::vector<MockQueryResult>; + +G_GNUC_UNUSED static std::ostream& +operator<<(std::ostream& os, const MockQueryResults& qrs) +{ + for (auto&& mi : qrs) + os << mi.query_match().thread_path << " :: " << mi.message_id().value_or("<none>") + << std::endl; + + return os; +} + +static void +calculate_threads(MockQueryResults& qres, bool descending) +{ + calculate_threads_real(qres, descending); +} + +using Expected = std::vector<std::pair<std::string, std::string>>; + +static void +assert_thread_paths(const MockQueryResults& qrs, const Expected& expected) +{ + for (auto&& exp : expected) { + auto it = std::find_if(qrs.begin(), qrs.end(), [&](auto&& qr) { + return qr.message_id().value_or("") == exp.first || + qr.path().value_or("") == exp.first; + }); + g_assert_true(it != qrs.end()); + mu_debug("thread-path ({}@{}): expected: '{}'; got '{}'", + it->message_id().value_or("<none>"), + it->path().value_or("<none>"), + exp.second, it->query_match().thread_path); + g_assert_cmpstr(exp.second.c_str(), ==, it->query_match().thread_path.c_str()); + } +} + +static void +test_sort_ascending() +{ + auto results = MockQueryResults{MockQueryResult{"m1", "1", {"m2"}}, + MockQueryResult{"m2", "2", {"m3"}}, + MockQueryResult{"m3", "3", {}}, + MockQueryResult{"m4", "4", {}}}; + + calculate_threads(results, false); + + assert_thread_paths(results, {{"m1", "0:0:0"}, {"m2", "0:0"}, {"m3", "0"}, {"m4", "1"}}); +} + +static void +test_sort_descending() +{ + auto results = MockQueryResults{MockQueryResult{"m1", "1", {"m2"}}, + MockQueryResult{"m2", "2", {"m3"}}, + MockQueryResult{"m3", "3", {}}, + MockQueryResult{"m4", "4", {}}}; + + calculate_threads(results, true); + + assert_thread_paths(results, + {{"m1", "1:f:f:z"}, {"m2", "1:f:z"}, {"m3", "1:z"}, {"m4", "0:z"}}); +} + +static void +test_id_table_inconsistent() +{ + auto results = MockQueryResults{ + MockQueryResult{"m1", "1", {"m2"}}, // 1->2 + MockQueryResult{"m2", "2", {"m1"}}, // 2->1 + MockQueryResult{"m3", "3", {"m3"}}, // self ref + MockQueryResult{"m4", "4", {"m3", "m5"}}, + MockQueryResult{"m5", "5", {"m4", "m4"}}, // dup parent + }; + + calculate_threads(results, false); + assert_thread_paths(results, + { + {"m2", "0"}, + {"m1", "0:0"}, + {"m3", "1"}, + {"m5", "1:0"}, + {"m4", "1:0:0"}, + }); +} + +static void +test_dups_dup_last() +{ + MockQueryResult r1{"m1", "1", {}}; + r1.query_match().flags |= QueryMatch::Flags::Leader; + r1.path_ = "/path1"; + + MockQueryResult r1_dup{"m1", "1", {}}; + r1_dup.query_match().flags |= QueryMatch::Flags::Duplicate; + r1_dup.path_ = "/path2"; + + auto results = MockQueryResults{r1, r1_dup}; + + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"/path1", "0"}, + {"/path2", "0:0"}, + }); +} + +static void +test_dups_dup_first() +{ + // now dup becomes the leader; this will _demote_ + // r1. + + MockQueryResult r1_dup{"m1", "1", {}}; + r1_dup.query_match().flags |= QueryMatch::Flags::Duplicate; + r1_dup.path_ = "/path1"; + + MockQueryResult r1{"m1", "1", {}}; + r1.query_match().flags |= QueryMatch::Flags::Leader; + r1.path_ = "/path2"; + + auto results = MockQueryResults{r1_dup, r1}; + + calculate_threads(results, false); + + assert_thread_paths(results, { + {"/path2", "0"}, + {"/path1", "0:0"}, + }); +} + +static void +test_dups_dup_multi() +{ + // now dup becomes the leader; this will _demote_ + // r1. + + MockQueryResult r1_dup1{"m1", "1", {}}; + r1_dup1.query_match().flags |= QueryMatch::Flags::Duplicate; + r1_dup1.path_ = "/path1"; + + MockQueryResult r1_dup2{"m1", "1", {}}; + r1_dup2.query_match().flags |= QueryMatch::Flags::Duplicate; + r1_dup2.path_ = "/path2"; + + MockQueryResult r1{"m1", "1", {}}; + r1.query_match().flags |= QueryMatch::Flags::Leader; + r1.path_ = "/path3"; + + auto results = MockQueryResults{r1_dup1, r1_dup2, r1}; + calculate_threads(results, false); + + assert_thread_paths(results, { + {"/path3", "0"}, + {"/path1", "0:0"}, + {"/path2", "0:1"}, + }); +} + + + + +static void +test_do_not_prune_root_empty_with_children() +{ + // m7 should not be nuked + auto results = MockQueryResults{ + MockQueryResult{"x1", "1", {"m7"}}, + MockQueryResult{"x2", "2", {"m7"}}, + }; + + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"x1", "0:0"}, + {"x2", "0:1"}, + }); +} + +static void +test_prune_root_empty_with_child() +{ + // m7 should be nuked + auto results = MockQueryResults{ + MockQueryResult{"m1", "1", {"m7"}}, + }; + + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"m1", "0"}, + }); +} + +static void +test_prune_empty_with_children() +{ + // m6 should be nuked + auto results = MockQueryResults{ + MockQueryResult{"m1", "1", {"m7", "m6"}}, + MockQueryResult{"m2", "2", {"m7", "m6"}}, + }; + + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"m1", "0:0"}, + {"m2", "0:1"}, + }); +} + +static void +test_thread_info_ascending() +{ + auto results = MockQueryResults{ + MockQueryResult{"m1", "5", {}}, + MockQueryResult{"m2", "1", {}}, + MockQueryResult{"m3", "3", {"m2"}}, + MockQueryResult{"m4", "2", {"m2"}}, + // orphan siblings + MockQueryResult{"m10", "6", {"m9"}}, + MockQueryResult{"m11", "7", {"m9"}}, + }; + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"m2", "0"}, // 2 + {"m4", "0:0"}, // 2 + {"m3", "0:1"}, // 3 + {"m1", "1"}, // 5 + + {"m10", "2:0"}, // 6 + {"m11", "2:1"}, // 7 + }); + + g_assert_true(results[0].query_match().has_flag(QueryMatch::Flags::Root)); + g_assert_true(results[1].query_match().has_flag(QueryMatch::Flags::Root | + QueryMatch::Flags::HasChild)); + g_assert_true(results[2].query_match().has_flag(QueryMatch::Flags::Last)); + g_assert_true(results[3].query_match().has_flag(QueryMatch::Flags::First)); + g_assert_true(results[4].query_match().has_flag(QueryMatch::Flags::Orphan | + QueryMatch::Flags::First)); + g_assert_true( + results[5].query_match().has_flag(QueryMatch::Flags::Orphan | QueryMatch::Flags::Last)); +} + +static void +test_thread_info_descending() +{ + auto results = MockQueryResults{ + MockQueryResult{"m1", "5", {}}, + MockQueryResult{"m2", "1", {}}, + MockQueryResult{"m3", "3", {"m2"}}, + MockQueryResult{"m4", "2", {"m2"}}, + // orphan siblings + MockQueryResult{"m10", "6", {"m9"}}, + MockQueryResult{"m11", "7", {"m9"}}, + }; + calculate_threads(results, true /*descending*/); + + assert_thread_paths(results, + { + {"m1", "1:z"}, // 5 + {"m2", "2:z"}, // 2 + {"m4", "2:f:z"}, // 2 + {"m3", "2:e:z"}, // 3 + + {"m10", "0:f:z"}, // 6 + {"m11", "0:e:z"}, // 7 + }); + g_assert_true(results[0].query_match().has_flag(QueryMatch::Flags::Root)); + g_assert_true(results[1].query_match().has_flag(QueryMatch::Flags::Root | + QueryMatch::Flags::HasChild)); + g_assert_true(results[2].query_match().has_flag(QueryMatch::Flags::Last)); + g_assert_true(results[3].query_match().has_flag(QueryMatch::Flags::First)); + + g_assert_true( + results[4].query_match().has_flag(QueryMatch::Flags::Orphan | QueryMatch::Flags::Last)); + g_assert_true(results[5].query_match().has_flag(QueryMatch::Flags::Orphan | + QueryMatch::Flags::First)); +} + +int +main(int argc, char* argv[]) +try { + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/threader/sort/ascending", test_sort_ascending); + g_test_add_func("/threader/sort/decending", test_sort_descending); + + g_test_add_func("/threader/id-table-inconsistent", test_id_table_inconsistent); + g_test_add_func("/threader/dups/dup-last", test_dups_dup_last); + g_test_add_func("/threader/dups/dup-first", test_dups_dup_first); + g_test_add_func("/threader/dups/dup-multi", test_dups_dup_multi); + + g_test_add_func("/threader/prune/do-not-prune-root-empty-with-children", + test_do_not_prune_root_empty_with_children); + g_test_add_func("/threader/prune/prune-root-empty-with-child", + test_prune_root_empty_with_child); + g_test_add_func("/threader/prune/prune-empty-with-children", + test_prune_empty_with_children); + + g_test_add_func("/threader/thread-info/ascending", test_thread_info_ascending); + g_test_add_func("/threader/thread-info/descending", test_thread_info_descending); + + return g_test_run(); +} catch (const std::runtime_error& re) { + std::cerr << re.what() << "\n"; + return 1; +} catch (...) { + std::cerr << "caught exception\n"; + return 1; +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query-threads.hh b/lib/mu-query-threads.hh new file mode 100644 index 0000000..5aab888 --- /dev/null +++ b/lib/mu-query-threads.hh @@ -0,0 +1,41 @@ +/* +** Copyright (C) 2021 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_QUERY_THREADS__ +#define MU_QUERY_THREADS__ + +#include "mu-query-results.hh" + +namespace Mu { +/** + * Calculate the threads for these query results; that is, determine the + * thread-paths for each message, so we can let Xapian order them in the correct + * order. + * + * Note - threads are sorted chronologically, and the messages below the top + * level are always sorted in ascending orde + * + * @param qres query results + * @param descending whether to sort the top-level in descending order + */ +void calculate_threads(QueryResults& qres, bool descending); + +} // namespace Mu + +#endif /*MU_QUERY_THREADS__*/ diff --git a/lib/mu-query-xapianizer.cc b/lib/mu-query-xapianizer.cc new file mode 100644 index 0000000..11aeee0 --- /dev/null +++ b/lib/mu-query-xapianizer.cc @@ -0,0 +1,521 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-query-parser.hh" + +#include <string_view> +#include <variant> +#include <array> +#include <type_traits> + +#include "utils/mu-option.hh" +#include <glib.h> +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +// backward compat +#ifndef HAVE_XAPIAN_FLAG_NGRAMS +#define FLAG_NGRAMS FLAG_CJK_NGRAM +#endif /*HAVE_XAPIAN_FLAG_NGRAMS*/ + +/** + * Expand terms for scripts without explicit word-breaks (e.g. + * Chinese/Japanese/Korean) in the way that Xapian expects it - + * use Xapian's built-in QueryParser just for that. + */ +static Result<Xapian::Query> +ngram_expand(const Field& field, const std::string& str) +{ + Xapian::QueryParser qp; + const auto pfx{std::string(1U, field.xapian_prefix())}; + + qp.set_default_op(Xapian::Query::OP_OR); + + return qp.parse_query(str, Xapian::QueryParser::FLAG_NGRAMS, pfx); +} + + +static Option<Sexp> +tail(Sexp&& s) +{ + if (!s.listp() || s.empty()) + return Nothing; + + s.list().erase(s.list().begin(), s.list().begin() + 1); + + return s; +} + +Option<std::string> +head_symbol(const Sexp& s) +{ + if (!s.listp() || s.empty() || !s.head() || !s.head()->symbolp()) + return Nothing; + + return s.head()->symbol().name; +} + + +Option<std::string> +string_nth(const Sexp& args, size_t n) +{ + if (!args.listp() || args.size() < n + 1) + return Nothing; + + if (auto&& item{args.list().at(n)}; !item.stringp()) + return Nothing; + else + return item.string(); +} + +static Result<Xapian::Query> +phrase(const Field& field, Sexp&& s) +{ + if (!field.is_phrasable_term()) + return Err(Error::Code::InvalidArgument, + "field {} does not support phrases", field.name); + + if (s.size() == 1 && s.front().stringp()) { + auto&& words{split(s.front().string(), " ")}; + std::vector<Xapian::Query> phvec; + phvec.reserve(words.size()); + for(auto&& w: words) + phvec.emplace_back(Xapian::Query{field.xapian_term(std::move(w))}); + return Xapian::Query{Xapian::Query::OP_PHRASE, + phvec.begin(), phvec.end()}; + } else + return Err(Error::Code::InvalidArgument, + "invalid phrase for field {}: '{}'", field.name, s.to_string()); +} + +static Result<Xapian::Query> +regex(const Store& store, const Field& field, const std::string& rx_str) +{ + auto&& str{utf8_flatten(rx_str)}; + auto&& rx{Regex::make(str, G_REGEX_OPTIMIZE)}; + if (!rx) { + mu_warning("invalid regexp: '{}': {}", str, rx.error().what()); + return Xapian::Query::MatchNothing; + } + + std::vector<Xapian::Query> rxvec; + store.for_each_term(field.id, [&](auto&& str) { + if (auto&& val{str.data() + 1}; rx->matches(val)) + rxvec.emplace_back(field.xapian_term(std::string_view{val})); + return true; + }); + + return Xapian::Query(Xapian::Query::OP_OR, rxvec.begin(), rxvec.end()); +} + + + +static Result<Xapian::Query> +range(const Field& field, Sexp&& s) +{ + auto&& r0{string_nth(s, 0)}; + auto&& r1{string_nth(s, 1)}; + if (!r0 || !r1) + return Err(Error::Code::InvalidArgument, "expected 2 range values"); + + // in the sexp, we use iso date/time for human readability; now convert to + // time_t + auto iso_to_lexnum=[](const std::string& s)->Option<std::string> { + if (s.empty()) + return s; + if (auto&& t{parse_date_time(s, true, true/*utc*/)}; !t) + return Nothing; + else + return to_lexnum(*t); + }; + + if (field == Field::Id::Date || field == Field::Id::Changed) { + // iso -> time_t + r0 = iso_to_lexnum(*r0); + r1 = iso_to_lexnum(*r1); + } else if (field == Field::Id::Size) { + if (!r0->empty()) + r0 = to_lexnum(::atoll(r0->c_str())); + if (!r1->empty()) + r1 = to_lexnum(::atoll(r1->c_str())); + } else + return Err(Error::Code::InvalidArgument, + "unsupported range field {}", field.name); + + if (r0->empty() && r1->empty()) + return Xapian::Query::MatchNothing; // empty range matches nothing. + else if (r0->empty() && !r1->empty()) + return Xapian::Query(Xapian::Query::OP_VALUE_LE, + field.value_no(), *r1); + else if (!r0->empty() && r1->empty()) + return Xapian::Query(Xapian::Query::OP_VALUE_GE, + field.value_no(), *r0); + else + return Xapian::Query(Xapian::Query::OP_VALUE_RANGE, + field.value_no(), *r0, *r1); +} + + + +using OpPair = std::pair<const std::string_view, Xapian::Query::op>; +static constexpr std::array<OpPair, 4> LogOpPairs = {{ + { "and", Xapian::Query::OP_AND }, + { "or", Xapian::Query::OP_OR }, + { "xor", Xapian::Query::OP_XOR }, + { "not", Xapian::Query::OP_AND_NOT } + }}; + +static Option<Xapian::Query::op> +find_log_op(const std::string& opname) +{ + for (auto&& p: LogOpPairs) + if (p.first == opname) + return p.second; + + return Nothing; +} + +static Result<Xapian::Query> parse(const Store& store, Sexp&& s, Mu::ParserFlags flags); + +static Result<Xapian::Query> +parse_logop(const Store& store, Xapian::Query::op op, Sexp&& args, Mu::ParserFlags flags) +{ + if (!args.listp() || args.empty()) + return Err(Error::Code::InvalidArgument, + "expected non-empty list but got", args.to_string()); + + std::vector<Xapian::Query> qs; + for (auto&& elm: args.list()) { + if (auto&& q{parse(store, std::move(elm), flags)}; !q) + return Err(std::move(q.error())); + else + qs.emplace_back(std::move(*q)); + } + + switch(op) { + case Xapian::Query::OP_AND_NOT: + // TODO: optimize AND_NOT + if (qs.size() != 1) + return Err(Error::Code::InvalidArgument, + "expected single argument for NOT"); + else + return Xapian::Query{op, Xapian::Query::MatchAll, qs.at(0)}; + + case Xapian::Query::OP_AND: + case Xapian::Query::OP_OR: + case Xapian::Query::OP_XOR: + return Xapian::Query(op, qs.begin(), qs.end()); + + default: + return Err(Error::Code::InvalidArgument, "unexpected xapian op"); + } +} + + +static Result<Xapian::Query> +parse_field_matcher(const Store& store, const Field& field, + const std::string& match_sym, Sexp&& args) +{ + auto&& str0{string_nth(args, 0)}; + + if (match_sym == wildcard_sym.name && str0) + return Xapian::Query{Xapian::Query::OP_WILDCARD, + field.xapian_term(*str0)}; + else if (match_sym == range_sym.name && !!str0) + return range(field, std::move(args)); + else if (match_sym == regex_sym.name && !!str0) + return regex(store, field, *str0); + else if (match_sym == phrase_sym.name) + return phrase(field, std::move(args)); + + return Err(Error::Code::InvalidArgument, + "invalid field '{}'/'{}' matcher: {}", + field.name, match_sym, args.to_string()); +} + +static Result<Xapian::Query> +parse_basic(const Field &field, Sexp &&vals, Mu::ParserFlags flags) +{ + auto ngrams = any_of(flags & ParserFlags::SupportNgrams); + if (!vals.stringp()) + return Err(Error::Code::InvalidArgument, "expected string"); + + auto&& val{vals.string()}; + + switch (field.id) { + case Field::Id::Flags: + if (auto&& finfo{flag_info(val)}; finfo) + return Xapian::Query{field.xapian_term(finfo->shortcut_lower())}; + else + return Err(Error::Code::InvalidArgument, "invalid flag '{}'", val); + case Field::Id::Priority: + if (auto&& prio{priority_from_name(val)}; prio) + return Xapian::Query{field.xapian_term(to_char(*prio))}; + else + return Err(Error::Code::InvalidArgument, "invalid priority '{}'", val); + default: { + auto q{Xapian::Query{field.xapian_term(val)}}; + if (ngrams) { // special case: cjk; see if we can create an expanded query. + if (field.is_phrasable_term() && contains_unbroken_script(val)) + if (auto&& ng{ngram_expand(field, val)}; ng) + return ng; + } + return q; + }} +} + +static Result<Xapian::Query> +parse(const Store& store, Sexp&& s, Mu::ParserFlags flags) +{ + auto&& headsym{head_symbol(s)}; + if (!headsym) + return Err(Error::Code::InvalidArgument, + "expected (symbol ...) but got {}", s.to_string()); + + // ie., something like (or|and| ... ....) + if (auto&& logop{find_log_op(*headsym)}; logop) { + if (auto&& args{tail(std::move(s))}; !args) + return Err(Error::Code::InvalidArgument, + "expected (logop ...) but got {}", + s.to_string()); + else + return parse_logop(store, *logop, std::move(*args), flags); + + } + // something like (field ...) + else if (auto&& field{field_from_name(*headsym)}; field) { + + auto&& rest{tail(std::move(s))}; + if (!rest || rest->empty()) + return Err(Error::Code::InvalidArgument, + "expected field-value or field-matcher"); + + auto&& matcher{rest->front()}; + // field-value: (field "value"); ensure "value" is there + if (matcher.stringp()) + return parse_basic(*field, std::move(matcher), flags); + + // otherwise, we expect a field-matcher, e.g. (field (phrase "a b c")) + // ensure the matcher is a list starting with a symbol + auto&& match_sym{head_symbol(matcher)}; + if (!match_sym) + return Err(Error::Code::InvalidArgument, + "expected field-matcher"); + + if (auto&& args{tail(std::move(matcher))}; !args) + return Err(Error::Code::InvalidArgument, "expected matcher arguments"); + else + return parse_field_matcher(store, *field, + *match_sym, std::move(*args)); + } + return Err(Error::Code::InvalidArgument, "unexpected sexp {}", s.to_string()); +} + +/* LCOV_EXCL_START*/ +// parse the way Xapian's internal parser does it; for testing. +static Xapian::Query +xapian_query_classic(const std::string& expr, Mu::ParserFlags flags) +{ + Xapian::QueryParser xqp; + + // add prefixes + field_for_each([&](auto&& field){ + + if (!field.is_searchable()) + return; + + const auto prefix{std::string(1U, field.xapian_prefix())}; + std::vector<std::string> names = { + std::string{field.name}, + std::string(1U, field.shortcut) + }; + if (!field.alias.empty()) + names.emplace_back(std::string{field.alias}); + + for (auto&& name: names) + xqp.add_prefix(name, prefix); + }); + + auto xflags = Xapian::QueryParser::FLAG_PHRASE | + Xapian::QueryParser::FLAG_BOOLEAN | + Xapian::QueryParser::FLAG_WILDCARD; + + if (any_of(flags & ParserFlags::SupportNgrams)) + xflags |= Xapian::QueryParser::FLAG_NGRAMS; + + xqp.set_default_op(Xapian::Query::OP_AND); + return xqp.parse_query(expr, xflags); +} +/* LCOV_EXCL_STOP*/ + +Result<Xapian::Query> +Mu::make_xapian_query(const Store& store, const std::string& expr, Mu::ParserFlags flags) noexcept +{ + if (any_of(flags & Mu::ParserFlags::XapianParser)) + return xapian_query_classic(expr, flags); + + return parse(store, Mu::parse_query(expr, true/*expand*/), flags); +} + + +#ifdef BUILD_XAPIANIZE_QUERY +int +main (int argc, char *argv[]) +{ + if (argc < 2) { + mu_printerrln("expected: parse-query <query>"); + return 1; + } + + auto store = Store::make(runtime_path(Mu::RuntimePath::XapianDb)); + if (!store) { + mu_printerrln("error: {}", store.error()); + return 2; + } + + std::string expr; + for (auto i = 1; i < argc; ++i) { + expr += argv[i]; + expr += " "; + } + + if (auto&& query{make_xapian_query(*store, expr)}; !query) { + mu_printerrln("error: {}", query.error()); + return 1; + } else + mu_println("mu: {}", query->get_description()); + + if (auto&& query{make_xapian_query(*store, expr, ParserFlags::XapianParser)}; !query) { + mu_printerrln("error: {}", query.error()); + return 2; + } else + mu_println("xp: {}", query->get_description()); + + return 0; + + +} +#endif /*BUILD_XAPIANIZE_QUERY*/ + +#if BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +using TestCase = std::pair<std::string, std::string>; + +static void +test_sexp() +{ + /* tail */ + g_assert_false(!!tail(Sexp{})); + auto t = tail(Sexp{1,2,3}); + g_assert_true(!!t && t->listp() && t->size() == 2); + + /* head_symbol */ + g_assert_false(!!head_symbol(Sexp{})); + assert_equal(head_symbol(Sexp{"foo"_sym, 1, 2}).value_or("bar"), "foo"); + + /* string_nth */ + g_assert_false(!!string_nth(Sexp{}, 123)); + g_assert_false(!!string_nth(Sexp{1, 2, 3}, 1)); + assert_equal(string_nth(Sexp{"aap", "noot", "mies"}, 2).value_or("wim"), "mies"); +} + + +static void +test_xapian() +{ + allow_warnings(); + + auto&& testhome{unwrap(make_temp_dir())}; + auto&& dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + auto&& store{unwrap(Store::make_new(dbpath, join_paths(testhome, "test-maildir")))}; + + // Xapian internal format (get_description()) is _not_ guaranteed + // to be the same between versions + auto&& zz{make_xapian_query(store, R"(subject:"hello world")")}; + assert_valid_result(zz); + /* LCOV_EXCL_START*/ + if (zz->get_description() != R"(Query((Shello world OR (Shello PHRASE 2 Sworld))))") { + mu_println("{}", zz->get_description()); + if (mu_test_mu_hacker()) { + // in the mu hacker case, we want to be warned if Xapian changed. + g_critical("xapian version mismatch"); + g_assert_true(false); + } else { + g_test_skip("incompatible xapian descriptions"); + return; + } + } + /* LCOV_EXCL_STOP*/ + + std::vector<TestCase> cases = { + + TestCase{R"(i:87h766tzzz.fsf@gnus.org)", R"(Query(I87h766tzzz.fsf@gnus.org))"}, + TestCase{R"(subject:foo to:bar)", R"(Query((Sfoo AND Tbar)))"}, + TestCase{R"(subject:"cuux*")", R"(Query(WILDCARD SYNONYM Scuux))"}, + TestCase{R"(subject:"hello world")", + R"(Query((Shello world OR (Shello PHRASE 2 Sworld))))"}, + TestCase{R"(subject:/boo/")", R"(Query())"}, + + // logic + TestCase{R"(not)", R"(Query((Tnot OR Cnot OR Hnot OR Fnot OR Snot OR Bnot OR Enot)))"}, + TestCase{R"(from:a and (from:b or from:c))", R"(Query((Fa AND (Fb OR Fc))))"}, + // optimize? + TestCase{R"(not from:a and to:b)", R"(Query(((<alldocuments> AND_NOT Fa) AND Tb)))"}, + TestCase{R"(cc:a not bcc:b)", R"(Query((Ca AND (<alldocuments> AND_NOT Hb))))"}, + + // ranges. + TestCase{R"(size:1..10")", R"(Query(VALUE_RANGE 17 g1 ga))"}, + TestCase{R"(size:10..1")", R"(Query(VALUE_RANGE 17 g1 ga))"}, + TestCase{R"(size:10..")", R"(Query(VALUE_GE 17 ga))"}, + TestCase{R"(size:..10")", R"(Query(VALUE_LE 17 ga))"}, + TestCase{R"(size:10")", R"(Query(VALUE_RANGE 17 ga ga))"}, // change? + TestCase{R"(size:..")", R"(Query())"}, + }; + + for (auto&& test: cases) { + auto&& xq{make_xapian_query(store, test.first)}; + assert_valid_result(xq); + assert_equal(xq->get_description(), test.second); + } + + remove_directory(testhome); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + Xapian::QueryParser qp; + + g_test_add_func("/query-parser/sexp", test_sexp); + g_test_add_func("/query-parser/xapianizer", test_xapian); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query.cc b/lib/mu-query.cc new file mode 100644 index 0000000..5b76005 --- /dev/null +++ b/lib/mu-query.cc @@ -0,0 +1,303 @@ +/* +** Copyright (C) 2008-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation; either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <mu-query.hh> + +#include <stdexcept> +#include <string> +#include <cctype> +#include <cstring> +#include <sstream> +#include <cmath> + +#include <stdlib.h> +#include <glib/gstdio.h> + +#include "mu-xapian-db.hh" +#include "mu-query-results.hh" +#include "mu-query-match-deciders.hh" +#include "mu-query-threads.hh" + +#include "mu-query-parser.hh" + +using namespace Mu; + +struct Query::Private { + Private(const Store& store) : + store_{store}, + parser_flags_{any_of(store_.message_options() & Message::Options::SupportNgrams) ? + ParserFlags::SupportNgrams : ParserFlags::None} {} + + Xapian::Enquire make_enquire(const std::string& expr, Field::Id sortfield_id, + QueryFlags qflags) const; + Xapian::Enquire make_related_enquire(const StringSet& thread_ids, + Field::Id sortfield_id, + QueryFlags qflags) const; + + Option<QueryResults> run_threaded(QueryResults&& qres, Xapian::Enquire& enq, + QueryFlags qflags, size_t max_size) const; + Option<QueryResults> run_singular(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const; + Option<QueryResults> run_related(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const; + + Option<QueryResults> run(const std::string& expr, + Field::Id sortfield_id, QueryFlags qflags, + size_t maxnum) const; + const Store& store_; + const ParserFlags parser_flags_; +}; + +Query::Query(const Store& store) : priv_{std::make_unique<Private>(store)} {} + +Query::~Query() = default; + +static Xapian::Enquire& +sort_enquire(Xapian::Enquire& enq, Field::Id sortfield_id, QueryFlags qflags) +{ + const auto value_no{field_from_id(sortfield_id).value_no()}; + enq.set_sort_by_value(value_no, any_of(qflags & QueryFlags::Descending)); + + return enq; +} + +static Xapian::Query +make_query(const Store& store, const std::string& expr, ParserFlags parser_flags) +{ + if (expr.empty() || expr == R"("")") + return Xapian::Query::MatchAll; + else { + if (auto&& q{make_xapian_query(store, expr, parser_flags)}; !q) { + mu_warning("error in query '{}': {}", expr, q.error().what()); + return Xapian::Query::MatchNothing; + } else + return q.value(); + } +} + +Xapian::Enquire +Query::Private::make_enquire(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags) const +{ + auto enq{store_.xapian_db().enquire()}; + enq.set_query(make_query(store_, expr, parser_flags_)); + sort_enquire(enq, sortfield_id, qflags); + + return enq; +} + +Xapian::Enquire +Query::Private::make_related_enquire(const StringSet& thread_ids, + Field::Id sortfield_id, + QueryFlags qflags) const +{ + auto enq{store_.xapian_db().enquire()}; + std::vector<Xapian::Query> qvec; + qvec.reserve(thread_ids.size()); + + for (auto&& t : thread_ids) + qvec.emplace_back(field_from_id(Field::Id::ThreadId).xapian_term(t)); + + Xapian::Query qr{Xapian::Query::OP_OR, qvec.begin(), qvec.end()}; + enq.set_query(qr); + + sort_enquire(enq, sortfield_id, qflags); + + return enq; +} + +struct ThreadKeyMaker : public Xapian::KeyMaker { + ThreadKeyMaker(const QueryMatches& matches) : match_info_(matches) {} + std::string operator()(const Xapian::Document& doc) const override { + const auto it{match_info_.find(doc.get_docid())}; + return (it == match_info_.end()) ? "" : it->second.thread_path; + } + const QueryMatches& match_info_; +}; + +Option<QueryResults> +Query::Private::run_threaded(QueryResults&& qres, Xapian::Enquire& enq, QueryFlags qflags, + size_t maxnum) const +{ + const auto descending{any_of(qflags & QueryFlags::Descending)}; + + calculate_threads(qres, descending); + + ThreadKeyMaker key_maker{qres.query_matches()}; + enq.set_sort_by_key(&key_maker, descending); + + DeciderInfo minfo; + minfo.matches = qres.query_matches(); + auto mset{enq.get_mset(0, maxnum, {}, make_thread_decider(qflags, minfo).get())}; + mset.fetch(); + + return QueryResults{mset, std::move(qres.query_matches())}; +} + +Option<QueryResults> +Query::Private::run_singular(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const +{ + // i.e. a query _without_ related messages, but still possibly + // with threading. + // + // In the threading case, the sortfield-id is ignored, we always sort by + // date (since threading the threading results are always by date.) + + const auto singular_qflags{qflags | QueryFlags::Leader}; + const auto threading{any_of(qflags & QueryFlags::Threading)}; + + DeciderInfo minfo{}; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wextra" + auto enq{make_enquire(expr, threading ? Field::Id::Date : sortfield_id, qflags)}; +#pragma GCC diagnostic ignored "-Wswitch-default" +#pragma GCC diagnostic pop + auto mset{enq.get_mset(0, maxnum, {}, + make_leader_decider(singular_qflags, minfo).get())}; + mset.fetch(); + + auto qres{QueryResults{mset, std::move(minfo.matches)}}; + + return threading ? run_threaded(std::move(qres), enq, qflags, maxnum) : qres; +} + +static Option<std::string> +opt_string(const Xapian::Document& doc, Field::Id id) noexcept +{ + const auto value_no{field_from_id(id).value_no()}; + std::string val = + xapian_try([&] { return doc.get_value(value_no); }, std::string{""}); + if (val.empty()) + return Nothing; + else + return Some(std::move(val)); +} + +Option<QueryResults> +Query::Private::run_related(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const +{ + // i.e. a query _with_ related messages and possibly with threading. + // + // In the threading case, the sortfield-id is ignored, we always sort by + // date (since threading the threading results are always by date.); + // moreover, in either threaded or non-threaded case, we sort the first + // ("leader") query by date, i.e, we prefer the newest or oldest + // (descending) messages. + const auto leader_qflags{QueryFlags::Leader | qflags}; + const auto threading{any_of(qflags & QueryFlags::Threading)}; + + // Run our first, "leader" query + DeciderInfo minfo{}; + auto enq{make_enquire(expr, Field::Id::Date, leader_qflags)}; + const auto mset{ + enq.get_mset(0, maxnum, {}, make_leader_decider(leader_qflags, minfo).get())}; + + // Gather the thread-ids we found + mset.fetch(); + minfo.thread_ids.reserve(mset.size()); + for (auto it = mset.begin(); it != mset.end(); ++it) + if (auto thread_id{opt_string(it.get_document(), Field::Id::ThreadId)}; thread_id) + minfo.thread_ids.emplace(std::move(*thread_id)); + + // Now, determine the "related query". + // + // In the threaded-case, we search among _all_ messages, since complete + // threads are preferred; no need to sort in that case since the search + // is unlimited and the sorting happens during threading. + auto r_enq = std::invoke([&]{ + if (threading) + return make_related_enquire(minfo.thread_ids, Field::Id::Date, + qflags); + else + return make_related_enquire(minfo.thread_ids, sortfield_id, qflags); + }); + + const auto r_mset{r_enq.get_mset(0, threading ? store_.size() : maxnum, {}, + make_related_decider(qflags, minfo).get())}; + auto qres{QueryResults{r_mset, std::move(minfo.matches)}}; + return threading ? run_threaded(std::move(qres), r_enq, qflags, maxnum) : qres; +} + +Option<QueryResults> +Query::Private::run(const std::string& expr, Field::Id sortfield_id, QueryFlags qflags, + size_t maxnum) const +{ + const auto eff_maxnum{maxnum == 0 ? store_.size() : maxnum}; + + if (any_of(qflags & QueryFlags::IncludeRelated)) + return run_related(expr, sortfield_id, qflags, eff_maxnum); + else + return run_singular(expr, sortfield_id, qflags, eff_maxnum); +} + +Result<QueryResults> +Query::run(const std::string& expr, Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const +{ + // some flags are for internal use only. + g_return_val_if_fail(none_of(qflags & QueryFlags::Leader), + Err(Error::Code::InvalidArgument, "cannot pass Leader flag")); + + StopWatch sw{ + mu_format("query: '{}'; (related:{}; threads:{}; ngrams:{}; max-size:{})", + expr, + any_of(qflags & QueryFlags::IncludeRelated) ? "yes" : "no", + any_of(qflags & QueryFlags::Threading) ? "yes" : "no", + any_of(priv_->parser_flags_ & ParserFlags::SupportNgrams) ? "yes" : "no", + maxnum == 0 ? std::string{"∞"} : std::to_string(maxnum))}; + + return xapian_try_result([&]{ + if (auto&& res = priv_->run(expr, sortfield_id, qflags, maxnum); res) + return Result<QueryResults>(Ok(std::move(res.value()))); + else + return Result<QueryResults>(Err(Error::Code::Query, + "failed to run query")); + }); +} + +size_t +Query::count(const std::string& expr) const +{ + return xapian_try( + [&] { + const auto enq{priv_->make_enquire(expr, {}, {})}; + auto mset{enq.get_mset(0, priv_->store_.size())}; + mset.fetch(); + return mset.size(); + }, + 0); +} + +/* LCOV_EXCL_START*/ +std::string +Query::parse(const std::string& expr, bool xapian) const +{ + if (xapian) + return make_query(priv_->store_, expr, + priv_->parser_flags_).get_description(); + else + return parse_query(expr).to_string(); +} +/* LCOV_EXCL_STOP*/ diff --git a/lib/mu-query.hh b/lib/mu-query.hh new file mode 100644 index 0000000..7ca1275 --- /dev/null +++ b/lib/mu-query.hh @@ -0,0 +1,100 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation; either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_QUERY_HH__ +#define __MU_QUERY_HH__ + +#include <memory> + +#include <glib.h> +#include <mu-store.hh> +#include <mu-query-results.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> +#include <utils/mu-result.hh> +#include <message/mu-message.hh> + +namespace Mu { + +class Query { +public: + /** + * Run a query on the store + * + * @param expr the search expression + * @param sortfield_id the sortfield-id. Default to Date + * @param flags query flags + * @param maxnum maximum number of results to return. 0 for 'no limit' + * + * @return the query-results or an error + */ + Result<QueryResults> run(const std::string& expr, + Field::Id sortfield_id = Field::Id::Date, + QueryFlags flags = QueryFlags::None, + size_t maxnum = 0) const; + + /** + * run a Xapian query to count the number of matches; for the syntax, please + * refer to the mu-query manpage + * + * @param expr the search expression; use "" to match all messages + * + * @return the number of matches + */ + size_t count(const std::string& expr = "") const; + + /** + * For debugging, get the internal string representation of the parsed + * query + * + * @param expr a xapian search expression + * @param xapian if true, show Xapian's internal representation, + * otherwise, mu's. + + * @return the string representation of the query + */ + std::string parse(const std::string& expr, bool xapian) const; + +private: + friend class Store; + + /** + * Construct a new Query instance. + * + * @param store a MuStore object + */ + Query(const Store& store); + /** + * DTOR + * + */ + ~Query(); + + /** + * Move CTOR + * + * @param other + */ + + struct Private; + std::unique_ptr<Private> priv_; +}; +} // namespace Mu + +#endif /*__MU_QUERY_HH__*/ diff --git a/lib/mu-scanner.cc b/lib/mu-scanner.cc new file mode 100644 index 0000000..bbc8d7e --- /dev/null +++ b/lib/mu-scanner.cc @@ -0,0 +1,425 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "mu-scanner.hh" + +#include "config.h" + +#include <chrono> +#include <mutex> +#include <atomic> +#include <thread> +#include <cstring> + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <glib.h> + +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-error.hh" + +using namespace Mu; + +using Mode = Scanner::Mode; + +/* + * dentry->d_ino, dentry->d_type may not be available + */ +struct dentry_t { + dentry_t(const struct dirent *dentry): +#if HAVE_DIRENT_D_INO + d_ino{dentry->d_ino}, +#endif /*HAVE_DIRENT_D_INO*/ + +#if HAVE_DIRENT_D_TYPE + d_type(dentry->d_type), +#endif /*HAVE_DIRENT_D_TYPE*/ + d_name{static_cast<const char*>(dentry->d_name)} {} +#if HAVE_DIRENT_D_INO + ino_t d_ino; +#endif /*HAVE_DIRENT_D_INO*/ + +#if HAVE_DIRENT_D_TYPE + unsigned char d_type; +#endif /*HAVE_DIRENT_D_TYPE*/ + + std::string d_name; +}; + +struct Scanner::Private { + Private(const std::string& root_dir, Scanner::Handler handler, Mode mode): + root_dir_{root_dir}, handler_{handler}, mode_{mode} { + if (root_dir_.length() > PATH_MAX) + throw Mu::Error{Error::Code::InvalidArgument, "path is too long"}; + if (!handler_) + throw Mu::Error{Error::Code::InvalidArgument, "missing handler"}; + } + ~Private() { stop(); } + + Result<void> start(); + void stop(); + + bool process_dentry(const std::string& path, const dentry_t& dentry, + bool is_maildir); + bool process_dir(const std::string& path, bool is_maildir); + + int lazy_stat(const char *fullpath, struct stat *stat_buf, + const dentry_t& dentry); + + bool maildirs_only_mode() const { return mode_ == Mode::MaildirsOnly; } + + const std::string root_dir_; + const Scanner::Handler handler_; + Mode mode_; + std::atomic<bool> running_{}; + std::mutex lock_; +}; + +static bool +ignore_dentry(const dentry_t& dentry) +{ + const auto d_name{dentry.d_name.c_str()}; + + /* dotdir? */ + if (d_name[0] == '\0' || (d_name[1] == '\0' && d_name[0] == '.') || + (d_name[2] == '\0' && d_name[0] == '.' && d_name[1] == '.')) + return true; + + if (d_name[0] != 't' && d_name[0] != 'h' && d_name[0] != '.') + return false; /* don't ignore */ + + if (::strcmp(d_name, "tmp") == 0 || ::strcmp(d_name, "hcache.db") == 0) + return true; // ignore + + if (d_name[0] == '.') + for (auto dname : { "nnmaildir", "notmuch", "noindex", "noupdate"}) + if (::strcmp(d_name + 1, dname) == 0) + return true; + + return false; /* don't ignore */ +} + + +/* + * stat() if necessary (we'd like to avoid it), which we can if we only need the + * file-type and we already have that from the dentry. + */ +int +Scanner::Private::lazy_stat(const char *path, struct stat *stat_buf, const dentry_t& dentry) +{ +#if HAVE_DIRENT_D_TYPE + if (maildirs_only_mode()) { + switch (dentry.d_type) { + case DT_REG: + stat_buf->st_mode = S_IFREG; + return 0; + case DT_DIR: + stat_buf->st_mode = S_IFDIR; + return 0; + default: + /* LNK is inconclusive; we need a stat. */ + break; + } + } +#endif /*HAVE_DIRENT_D_TYPE*/ + + int res = ::stat(path, stat_buf); + if (res != 0) + mu_warning("failed to stat {}: {}", path, g_strerror(errno)); + + return res; +} + + +bool +Scanner::Private::process_dentry(const std::string& path, const dentry_t& dentry, + bool is_maildir) +{ + if (ignore_dentry(dentry)) + return true; + + auto call_handler=[&](auto&& path, auto&& statbuf, auto&& htype)->bool { + return maildirs_only_mode() ? true : handler_(path, statbuf, htype); + }; + + const auto fullpath{join_paths(path, dentry.d_name)}; + struct stat statbuf{}; + if (lazy_stat(fullpath.c_str(), &statbuf, dentry) != 0) + return false; + + if (maildirs_only_mode() && S_ISDIR(statbuf.st_mode) && dentry.d_name == "cur") { + handler_(path/*without cur*/, {}, Scanner::HandleType::Maildir); + return true; // found maildir; no need to recurse further. + } + + if (S_ISDIR(statbuf.st_mode)) { + const auto new_cur = dentry.d_name == "cur" || dentry.d_name == "new"; + const auto htype = + new_cur ? + Scanner::HandleType::EnterNewCur : + Scanner::HandleType::EnterDir; + + const auto res = call_handler(fullpath, &statbuf, htype); + if (!res) + return true; // skip + + process_dir(fullpath, new_cur); + return call_handler(fullpath, &statbuf, Scanner::HandleType::LeaveDir); + + } else if (S_ISREG(statbuf.st_mode) && is_maildir) + return call_handler(fullpath, &statbuf, Scanner::HandleType::File); + + mu_debug("skip {} (neither maildir-file nor directory)", fullpath); + + return true; +} + +bool +Scanner::Private::process_dir(const std::string& path, bool is_maildir) +{ + if (!running_) + return true; /* we're done */ + + if (G_UNLIKELY(path.length() > PATH_MAX)) { + // note: unlikely to hit this, one case would be a self-referential + // symlink; that should be caught earlier, so this is just a backstop. + mu_warning("path is too long: {}", path); + return false; + } + + const auto dir{::opendir(path.c_str())}; + if (G_UNLIKELY(!dir)) { + mu_warning("failed to scan dir {}: {}", path, g_strerror(errno)); + return false; + } + + std::vector<dentry_t> dir_entries; + while (running_) { + errno = 0; + if (const auto& dentry{::readdir(dir)}; dentry) { +#if HAVE_DIRENT_D_TYPE /* optimization: filter out non-dirs early. NB not all file-systems support + * returning the file-type in `d_type`, so don't skip `DT_UNKNOWN`. + */ + if (maildirs_only_mode() && + dentry->d_type != DT_DIR && + dentry->d_type != DT_LNK && + dentry->d_type != DT_UNKNOWN) + continue; +#endif /*HAVE_DIRENT_D_TYPE*/ + dir_entries.emplace_back(dentry); + continue; + } else if (errno != 0) { + mu_warning("failed to read {}: {}", path, g_strerror(errno)); + continue; + } + + break; + } + ::closedir(dir); + +#if HAVE_DIRENT_D_INO + // sort by i-node; much faster on rotational (HDDs) devices and on SSDs + // sort is quick enough to not matter much + std::sort(dir_entries.begin(), dir_entries.end(), + [](auto&& d1, auto&& d2){ return d1.d_ino < d2.d_ino; }); +#endif /*HAVEN_DIRENT_D_INO*/ + + // now process... + for (auto&& dentry: dir_entries) + process_dentry(path, dentry, is_maildir); + + return true; +} + +Result<void> +Scanner::Private::start() +{ + const auto mode{F_OK | R_OK}; + if (G_UNLIKELY(::access(root_dir_.c_str(), mode) != 0)) + return Err(Error::Code::File, "'{}' is not readable: {}", root_dir_, + g_strerror(errno)); + + struct stat statbuf {}; + if (G_UNLIKELY(::stat(root_dir_.c_str(), &statbuf) != 0)) + return Err(Error::Code::File, "'{}' is not stat'able: {}", + root_dir_, g_strerror(errno)); + + if (G_UNLIKELY(!S_ISDIR(statbuf.st_mode))) + return Err(Error::Code::File, + "'{}' is not a directory", root_dir_); + + running_ = true; + mu_debug("starting scan @ {}", root_dir_); + + const auto bname{basename(root_dir_)}; + const auto is_maildir = bname == "cur" || bname == "new"; + + const auto start{std::chrono::steady_clock::now()}; + process_dir(root_dir_, is_maildir); + const auto elapsed = std::chrono::steady_clock::now() - start; + mu_debug("finished scan of {} in {} ms", root_dir_, to_ms(elapsed)); + running_ = false; + + return Ok(); +} + +void +Scanner::Private::stop() +{ + if (running_) { + mu_debug("stopping scan"); + running_ = false; + } +} + +Scanner::Scanner(const std::string& root_dir, Scanner::Handler handler, Mode flavor) + : priv_{std::make_unique<Private>(root_dir, handler, flavor)} +{} + +Scanner::~Scanner() = default; + +Result<void> +Scanner::start() +{ + if (priv_->running_) + return Ok(); // nothing to do + + auto res = priv_->start(); /* blocks */ + priv_->running_ = false; + + return res; +} + +void +Scanner::stop() +{ + std::lock_guard l(priv_->lock_); + priv_->stop(); +} + +bool +Scanner::is_running() const +{ + return priv_->running_; +} + + +#if BUILD_TESTS +/* LCOV_EXCL_START*/ +#include "mu-test-utils.hh" + +static void +test_scan_maildirs() +{ + allow_warnings(); + + size_t count{}; + Scanner scanner{ + MU_TESTMAILDIR, + [&](const std::string& fullpath, const struct stat* statbuf, auto&& htype) -> bool { + ++count; + g_usleep(10000); + return true; + }}; + assert_valid_result(scanner.start()); + scanner.stop(); + count = 0; + assert_valid_result(scanner.start()); + + while (scanner.is_running()) { g_usleep(100000); } + + // very rudimentary test... + g_assert_cmpuint(count,==,23); +} + +static void +test_count_maildirs() +{ + allow_warnings(); + + std::vector<std::string> dirs; + Scanner scanner{ + MU_TESTMAILDIR2, + [&](const std::string& fullpath, const struct stat* statbuf, auto&& htype) -> bool { + dirs.emplace_back(basename(fullpath)); + return true; + }, Scanner::Mode::MaildirsOnly}; + assert_valid_result(scanner.start()); + + while (scanner.is_running()) { g_usleep(1000); } + + g_assert_cmpuint(dirs.size(),==,3); + g_assert_true(seq_find_if(dirs, [](auto& p){return p == "bar";}) != dirs.end()); + g_assert_true(seq_find_if(dirs, [](auto& p){return p == "Foo";}) != dirs.end()); + g_assert_true(seq_find_if(dirs, [](auto& p){return p == "wom_bat";}) != dirs.end()); +} + +static void +test_fail_nonexistent() +{ + allow_warnings(); + + Scanner scanner{"/foo/bar/non-existent", + [&](auto&& a1, auto&& a2, auto&& a3){ return false; }}; + g_assert_false(scanner.is_running()); + g_assert_false(!!scanner.start()); + g_assert_false(scanner.is_running()); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/scanner/scan-maildirs", test_scan_maildirs); + g_test_add_func("/scanner/count-maildirs", test_count_maildirs); + g_test_add_func("/scanner/fail-nonexistent", test_fail_nonexistent); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ + +#if BUILD_LIST_MAILDIRS + +static bool +on_path(const std::string& path, struct stat* statbuf, Scanner::HandleType htype) +{ + mu_println("{}", path); + return true; +} + +int +main (int argc, char *argv[]) +{ + if (argc < 2) { + mu_printerrln("expected: path to maildir"); + return 1; + } + + Scanner scanner{argv[1], on_path, Mode::MaildirsOnly}; + + scanner.start(); + + return 0; +} +/* LCOV_EXCL_STOP*/ +#endif /*BUILD_LIST_MAILDIRS*/ diff --git a/lib/mu-scanner.hh b/lib/mu-scanner.hh new file mode 100644 index 0000000..e124c52 --- /dev/null +++ b/lib/mu-scanner.hh @@ -0,0 +1,122 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_SCANNER_HH__ +#define MU_SCANNER_HH__ + +#include <functional> +#include <memory> +#include <utils/mu-result.hh> + +#include <dirent.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +namespace Mu { + +/** + * @brief Maildir scanner + * + * Scans maildir (trees) recursively, and calls the Handler callback for + * directories & files. + * + * It filters out (i.e., does *not* call the handler for): + * - files starting with '.' + * - files that do not live in a cur / new leaf maildir + * - directories '.' and '..' and 'tmp' +*/ +class Scanner { + public: + enum struct HandleType { + /* + * Mode: All + */ + File, + EnterNewCur, /* cur/ or new/ */ + EnterDir, /* some other directory */ + LeaveDir, + /* + * Mode: Maildir + */ + Maildir, + }; + + /** + * Callback handler function + * + * path: full file-system path + * statbuf: stat result or nullptr (for Mode::MaildirsOnly) + * htype: HandleType. For Mode::MaildirsOnly only Maildir + */ + using Handler = std::function< + bool(const std::string& path, struct stat* statbuf, HandleType htype)>; + + /** + * Running mode for this Scanner + */ + enum struct Mode { + All, /**< Vanilla */ + MaildirsOnly /**< Only return maildir to handler */ + }; + + /** + * Construct a scanner object for scanning a directory, recursively. + * + * If handler is a directory + * + * @param root_dir root dir to start scanning + * @param handler handler function for some direntry + * @param options options to influence behavior + */ + Scanner(const std::string& root_dir, Handler handler, Mode mode = Mode::All); + + /** + * DTOR + */ + ~Scanner(); + + /**# + * Start the scan; this is a blocking call than runs until + * finished or (from another thread) stop() is called. + * + * @return Ok if starting worked; an Error otherwise + */ + Result<void> start(); + + /** + * Request stopping the scan if it's running; otherwise do nothing + */ + void stop(); + + /** + * Is a scan currently running? + * + * @return true or false + */ + bool is_running() const; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; + +} // namespace Mu + +#endif /* MU_SCANNER_HH__ */ diff --git a/lib/mu-script.cc b/lib/mu-script.cc new file mode 100644 index 0000000..81d481b --- /dev/null +++ b/lib/mu-script.cc @@ -0,0 +1,162 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "mu-script.hh" +#include "mu/mu-options.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-option.hh" + +#include <fstream> +#include <iostream> + +#ifdef BUILD_GUILE +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wredundant-decls" +#include <libguile.h> +#pragma GCC diagnostic pop +#endif /*BUILD_GUILE*/ + +using namespace Mu; + +static std::string +get_name(const std::string& path) +{ + auto pos = path.find_last_of("/"); + if (pos == std::string::npos) + return path; + + auto name = path.substr(pos + 1); + + pos = name.find_last_of("."); + if (pos == std::string::npos) + return name; + + return name.substr(0, pos); +} + + +static Mu::Option<Mu::ScriptInfo> +get_info(std::string&& path, const std::string& prefix) +{ + std::ifstream file{path}; + if (!file.is_open()) { + mu_warning ("failed to open {}", path); + return Nothing; + } + + Mu::ScriptInfo info{}; + info.path = path; + info.name = get_name(path); + + std::string line; + while (std::getline(file, line)) { + + if (line.find(prefix) != 0) + continue; + + line = line.substr(prefix.length()); + + if (info.oneline.empty()) + info.oneline = line; + else + info.description += line; + } + + // std::cerr << "ONELINE: " << info.oneline << '\n'; + // std::cerr << "DESCR : " << info.description << '\n'; + + return info; +} + + + +static void +script_infos_in_dir(const std::string& scriptdir, Mu::ScriptInfos& infos) +{ + DIR *dir = opendir(scriptdir.c_str()); + if (!dir) { + mu_debug("failed to open '{}': {}", scriptdir, + g_strerror(errno)); + return; + } + + const std::string ext{".scm"}; + + struct dirent *dentry; + while ((dentry = readdir(dir))) { + + if (!g_str_has_suffix(dentry->d_name, ext.c_str())) + continue; + + auto&& info = get_info(scriptdir + "/" + dentry->d_name, ";; INFO: "); + if (!info) + continue; + + infos.emplace_back(std::move(*info)); + } + + closedir(dir); /* ignore error checking... */ +} + + +Mu::ScriptInfos +Mu::script_infos(const Mu::ScriptPaths& paths) +{ + /* create a list of names, paths */ + ScriptInfos infos; + for (auto&& dir: paths) { + script_infos_in_dir(dir, infos); + } + + std::sort(infos.begin(), infos.end(), [](auto&& i1, auto&& i2) { + return i1.name < i2.name; + }); + + return infos; +} + +Result<void> +Mu::run_script(const std::string& path, + const std::vector<std::string>& args) +{ +#ifndef BUILD_GUILE + return Err(Error::Code::Script, + "guile script support is not available"); +#else + std::string mainargs; + for (auto&& arg: args) + mainargs += mu_format("{}\"{}\"", mainargs.empty() ? "" : " ", arg); + auto expr = mu_format("(main '(\"{}\" {}))", get_name(path), mainargs); + + std::vector<const char*> argv = { + GUILE_BINARY, + "-l", path.c_str(), + "-c", expr.c_str(), + }; + + /* does not return */ + scm_boot_guile(argv.size(), const_cast<char**>(argv.data()), + [](void *closure, int argc, char **argv) { + scm_shell(argc, argv); + }, NULL); + + return Ok(); +#endif /*BUILD_GUILE*/ +} diff --git a/lib/mu-script.hh b/lib/mu-script.hh new file mode 100644 index 0000000..48ff45a --- /dev/null +++ b/lib/mu-script.hh @@ -0,0 +1,65 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#ifndef MU_SCRIPT_HH__ +#define MU_SCRIPT_HH__ + +#include <string> +#include <vector> + +#include <utils/mu-result.hh> + +namespace Mu { + +/** + * Information about a script. + * + */ +struct ScriptInfo { + std::string name; /**< Name of script */ + std::string path; /**< Full path to script */ + std::string oneline; /**< One-line description */ + std::string description; /**< More help */ +}; + +/// Sequence of script infos. +using ScriptInfos = std::vector<ScriptInfo>; + +/** + * Get information about the available scripts + * + * @return infos + */ +using ScriptPaths = std::vector<std::string>; +ScriptInfos script_infos(const ScriptPaths& paths); + + +/** + * Run some specific script + * + * @param path full path to the scripts + * @param args argument vector to pass to the script + * + * @return Ok() or some error; however, note that this does not return after succesfully + * starting a script. + */ +Result<void> run_script(const std::string& path, const std::vector<std::string>& args); + +} // namepace Mu + +#endif /* MU_SCRIPT_HH__ */ diff --git a/lib/mu-server.cc b/lib/mu-server.cc new file mode 100644 index 0000000..62c9ca0 --- /dev/null +++ b/lib/mu-server.cc @@ -0,0 +1,1087 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-server.hh" + +#include "message/mu-message.hh" + +#include <fstream> +#include <sstream> +#include <string> +#include <algorithm> +#include <atomic> +#include <thread> +#include <mutex> +#include <variant> +#include <functional> + +#include <cstring> +#include <glib.h> +#include <glib/gprintf.h> +#include <unistd.h> + +#include "mu-maildir.hh" +#include "mu-query.hh" +#include "mu-store.hh" + +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" + +#include "utils/mu-option.hh" +#include "utils/mu-command-handler.hh" +#include "utils/mu-readline.hh" + +using namespace Mu; + +/* LCOV_EXCL_START */ + +/// output stream to _either_ a file or to a stringstream +struct OutputStream { + /** + * Construct an OutputStream for a tempfile + * + * @param tmp_dir dir for temp files + */ + OutputStream(const std::string& tmp_dir): + fname_{join_paths(tmp_dir, + mu_format("mu-{}.eld", g_get_monotonic_time()))}, + out_{std::ofstream{fname_}} { + if (!out().good()) + throw Mu::Error{Error::Code::File, "failed to create temp-file"}; + } + /** + * Construct an OutputStream for a stringstream + * + * @param cdr name of the output (e.g., "contacts") + * + * @return + */ + OutputStream(): out_{std::ostringstream{}} {} + + /** + * Get a writable ostream + * + * @return an ostream + */ + std::ostream& out() { + if (std::holds_alternative<std::ofstream>(out_)) + return std::get<std::ofstream>(out_); + else + return std::get<std::ostringstream>(out_); + } + + /// conversion + operator std::ostream&() { return out(); } + + /** + * Get the output as a string, either something like, either a lisp form + * or a the full path to a temp file containing the same. + * + * @return lisp form or path + */ + std::string to_string() const { + return std::holds_alternative<std::ostringstream>(out_) ? + std::get<std::ostringstream>(out_).str() : + quote(fname_); + } + + /** + * Delete file, if any. Only do this when the OutputStream is no + * longer needed. + */ + void unlink () { + if (fname_.empty()) + return; + if (auto&&res{::unlink(fname_.c_str())}; res != 0) + mu_warning("failed to unlink '{}'", ::strerror(res)); + else + mu_debug("unlinked output-stream {}", fname_); + } + +private: + std::string fname_; + using OutType = std::variant<std::ofstream, std::ostringstream>; + OutType out_; +}; + + + +/// @brief object to manage the server-context for all commands. +struct Server::Private { + Private(Store& store, const Server::Options& opts, Output output) + : store_{store}, options_{opts}, output_{output}, + command_handler_{make_command_map()}, + keep_going_{true}, + tmp_dir_{unwrap(make_temp_dir())} + {} + + ~Private() { + indexer().stop(); + if (index_thread_.joinable()) + index_thread_.join(); + if (!tmp_dir_.empty()) + remove_directory(tmp_dir_); + } + // + // construction helpers + // + CommandHandler::CommandInfoMap make_command_map(); + + // + // acccessors + Store& store() { return store_; } + const Store& store() const { return store_; } + Indexer& indexer() { return store().indexer(); } + //CommandMap& command_map() const { return command_map_; } + + // + // invoke + // + bool invoke(const std::string& expr) noexcept; + + // + // output + void output(const std::string& str, Server::OutputFlags flags = {}) const { + if (output_) + output_(str, flags); + } + void output_sexp(const Sexp& sexp, Server::OutputFlags flags = {}) const { + output(sexp.to_string(), flags); + } + + size_t output_results(const QueryResults& qres, size_t batch_size) const; + + // + // handlers for various commands. + // + void add_handler(const Command& cmd); + void compose_handler(const Command& cmd); + void contacts_handler(const Command& cmd); + void data_handler(const Command& cmd); + void find_handler(const Command& cmd); + void help_handler(const Command& cmd); + void index_handler(const Command& cmd); + void move_handler(const Command& cmd); + void mkdir_handler(const Command& cmd); + void ping_handler(const Command& cmd); + void queries_handler(const Command& cmd); + void quit_handler(const Command& cmd); + void remove_handler(const Command& cmd); + void view_handler(const Command& cmd); + +private: + void move_docid(Store::Id docid, Option<std::string> flagstr, + bool new_name, bool no_view); + + void perform_move(Store::Id docid, + const Message& msg, + const std::string& maildirarg, + Flags flags, + bool new_name, + bool no_view); + + void view_mark_as_read(Store::Id docid, Message&& msg, bool rename); + + OutputStream make_output_stream() const { + if (options_.allow_temp_file) + return OutputStream{tmp_dir_}; + else + return OutputStream{}; + } + + std::ofstream make_temp_file_stream(std::string& fname) const; + + Store& store_; + Server::Options options_; + Server::Output output_; + const CommandHandler command_handler_; + std::atomic<bool> keep_going_{}; + std::thread index_thread_; + std::string tmp_dir_; +}; + +static void +append_metadata(std::string& str, const QueryMatch& qmatch) +{ + const auto td{::atoi(qmatch.thread_date.c_str())}; + + str += mu_format(" :meta (:path \"{}\" :level {} :date \"{}\" " + ":data-tstamp ({} {} 0)", + qmatch.thread_path, + qmatch.thread_level, + qmatch.thread_date, + static_cast<unsigned>(td >> 16), + static_cast<unsigned>(td & 0xffff)); + + if (qmatch.has_flag(QueryMatch::Flags::Root)) + str += " :root t"; + if (qmatch.has_flag(QueryMatch::Flags::Related)) + str += " :related t"; + if (qmatch.has_flag(QueryMatch::Flags::First)) + str += " :first-child t"; + if (qmatch.has_flag(QueryMatch::Flags::Last)) + str += " :last-child t"; + if (qmatch.has_flag(QueryMatch::Flags::Orphan)) + str += " :orphan t"; + if (qmatch.has_flag(QueryMatch::Flags::Duplicate)) + str += " :duplicate t"; + if (qmatch.has_flag(QueryMatch::Flags::HasChild)) + str += " :has-child t"; + if (qmatch.has_flag(QueryMatch::Flags::ThreadSubject)) + str += " :thread-subject t"; + + str += ')'; +} + +/* + * A message here consists of a message s-expression with optionally a :docid + * and/or :meta expression added. + * + * We could parse the sexp and use the Sexp APIs to add some things... but... + * it's _much_ faster to directly work on the string representation: remove the + * final ')', add a few items, and add the ')' again. + */ +static std::string +msg_sexp_str(const Message& msg, Store::Id docid, const Option<QueryMatch&> qm) +{ + auto&& sexpstr{msg.document().sexp_str()}; + + if (docid != 0 || qm) { + sexpstr.reserve(sexpstr.size () + (docid == 0 ? 0 : 16) + (qm ? 64 : 0)); + + // remove the closing ( ... ) + sexpstr.erase(sexpstr.end() - 1); + + if (docid != 0) + sexpstr += " :docid " + to_string(docid); + if (qm) + append_metadata(sexpstr, *qm); + + sexpstr += ')'; // ... end close it again. + } + + return sexpstr; +} + + +CommandHandler::CommandInfoMap +Server::Private::make_command_map() +{ + CommandHandler::CommandInfoMap cmap; + + using CommandInfo = CommandHandler::CommandInfo; + using ArgMap = CommandHandler::ArgMap; + using ArgInfo = CommandHandler::ArgInfo; + using Type = Sexp::Type; + using Type = Sexp::Type; + + cmap.emplace( + "add", + CommandInfo{ + ArgMap{{":path", ArgInfo{Type::String, true, "file system path to the message"}}}, + "add a message to the store", + [&](const auto& params) { add_handler(params); }}); + + cmap.emplace( + "contacts", + CommandInfo{ + ArgMap{{":personal", ArgInfo{Type::Symbol, false, "only personal contacts"}}, + {":after", + ArgInfo{Type::String, false, "only contacts seen after time_t string"}}, + {":tstamp", ArgInfo{Type::String, false, "return changes since tstamp"}}, + {":maxnum", ArgInfo{Type::Number, false, "max number of contacts to return"}} + }, + "get contact information", + [&](const auto& params) { contacts_handler(params); }}); + + cmap.emplace( + "data", + CommandInfo{ + ArgMap{{":kind", ArgInfo{Type::Symbol, true, "kind of data (maildirs)"}}}, + "request data of some kind", + [&](const auto& params) { data_handler(params); }}); + + cmap.emplace( + "find", + CommandInfo{ + ArgMap{{":query", ArgInfo{Type::String, true, "search expression"}}, + {":threads", + ArgInfo{Type::Symbol, false, "whether to include threading information"}}, + {":sortfield", ArgInfo{Type::Symbol, false, "the field to sort results by"}}, + {":descending", + ArgInfo{Type::Symbol, false, "whether to sort in descending order"}}, + {":batch-size", ArgInfo{Type::Number, false, "batch size for result"}}, + {":maxnum", ArgInfo{Type::Number, false, "maximum number of result (hint)"}}, + {":skip-dups", + ArgInfo{Type::Symbol, + false, + "whether to skip messages with duplicate message-ids"}}, + {":include-related", + ArgInfo{Type::Symbol, + false, + "whether to include other message related to matching ones"}}}, + "query the database for messages", + [&](const auto& params) { find_handler(params); }}); + + cmap.emplace( + "help", + CommandInfo{ + ArgMap{{":command", ArgInfo{Type::Symbol, false, "command to get information for"}}, + {":full", ArgInfo{Type::Symbol, false, "show full descriptions"}}}, + "get information about one or all commands", + [&](const auto& params) { help_handler(params); }}); + cmap.emplace( + "index", + CommandInfo{ + ArgMap{{":my-addresses", ArgInfo{Type::List, false, "list of 'my' addresses"}}, + {":cleanup", + ArgInfo{Type::Symbol, + false, + "whether to remove stale messages from the store"}}, + {":lazy-check", + ArgInfo{Type::Symbol, + false, + "whether to avoid indexing up-to-date directories"}}}, + "scan maildir for new/updated/removed messages", + [&](const auto& params) { index_handler(params); }}); + cmap.emplace( + "mkdir", + CommandInfo{ + ArgMap{ + {":path", ArgInfo{Type::String, true, "location for the new maildir"}}, + {":update", ArgInfo{Type::Symbol, false, + "whether to send an update after creating"}} + }, "create a new maildir", + [&](const auto& params) { mkdir_handler(params); }}); + cmap.emplace( + "move", + CommandInfo{ + ArgMap{ + {":docid", ArgInfo{Type::Number, false, "document-id"}}, + {":msgid", ArgInfo{Type::String, false, "message-id"}}, + {":flags", ArgInfo{Type::String, false, "new flags for the message"}}, + {":maildir", ArgInfo{Type::String, false, "the target maildir"}}, + {":rename", ArgInfo{Type::Symbol, false, "change filename when moving"}}, + {":no-view", + ArgInfo{Type::Symbol, false, "if set, do not hint at updating the view"}}, + }, + "move messages and/or change their flags", + [&](const auto& params) { move_handler(params); }}); + cmap.emplace( + "ping", + CommandInfo{ + ArgMap{}, + "ping the mu-server and get server information in the response", + [&](const auto& params) { ping_handler(params); }}); + + cmap.emplace( + "queries", + CommandInfo{ + ArgMap{ + {":queries", + ArgInfo{Type::List, false, "queries for which to get read/unread numbers"}}, + }, + "get unread/totals information for a list of queries", + [&](const auto& params) { queries_handler(params); }}); + + cmap.emplace("quit", CommandInfo{{}, "quit the mu server", [&](const auto& params) { + quit_handler(params); + }}); + + cmap.emplace( + "remove", + CommandInfo{ + ArgMap{{":docid", + ArgInfo{Type::Number, false, "document-id for the message to remove"}}, + {":path", + ArgInfo{Type::String, false, "document-id for the message to remove"}} + }, + "remove a message from filesystem and database, using either :docid or :path", + [&](const auto& params) { remove_handler(params); }}); + + cmap.emplace( + "view", + CommandInfo{ArgMap{ + {":docid", ArgInfo{Type::Number, false, "document-id"}}, + {":msgid", ArgInfo{Type::String, false, "message-id"}}, + {":path", ArgInfo{Type::String, false, "message filesystem path"}}, + {":mark-as-read", + ArgInfo{Type::Symbol, false, "mark message as read (if not already)"}}, + {":rename", ArgInfo{Type::Symbol, false, "change filename when moving"}}, + }, + "view a message. exactly one of docid/msgid/path must be specified", + [&](const auto& params) { view_handler(params); }}); + return cmap; +} + +bool +Server::Private::invoke(const std::string& expr) noexcept +{ + auto make_error=[](auto&& code, auto&& msg) { + return Sexp().put_props( + ":error", Error::error_number(code), + ":message", msg); + }; + + if (!keep_going_) + return false; + try { + auto cmd{Command::make_parse(std::string{expr})}; + if (!cmd) + throw cmd.error(); + + auto res = command_handler_.invoke(*cmd); + if (!res) + throw res.error(); + + } catch (const Mu::Error& me) { + output_sexp(make_error(me.code(), mu_format("{}", + me.what()))); + keep_going_ = true; + } catch (const Xapian::Error& xerr) { + output_sexp(make_error(Error::Code::Internal, + mu_format("xapian error: {}: {}", + xerr.get_type(), xerr.get_description()))); + keep_going_ = false; + } catch (const std::runtime_error& re) { + output_sexp(make_error(Error::Code::Internal, + mu_format("caught runtime exception: {}", + re.what()))); + keep_going_ = false; + } catch (const std::out_of_range& oore) { + output_sexp(make_error(Error::Code::Internal, + mu_format("caught out-of-range exception: {}", + oore.what()))); + keep_going_ = false; + } catch (const std::exception& e) { + output_sexp(make_error(Error::Code::Internal, + mu_format(" exception: {}", e.what()))); + keep_going_ = false; + } catch (...) { + output_sexp(make_error(Error::Code::Internal, + mu_format("something went wrong: quitting"))); + keep_going_ = false; + } + + return keep_going_; +} + +/* 'add' adds a message to the database, and takes two parameters: 'path', which + * is the full path to the message, and 'maildir', which is the maildir this + * message lives in (e.g. "/inbox"). + * + * responds with an (added . <message sexp>) forr the new message + */ +void +Server::Private::add_handler(const Command& cmd) +{ + auto path{cmd.string_arg(":path")}; + const auto docid_res{store().add_message(*path)}; + + if (!docid_res) + throw docid_res.error(); + + const auto docid{docid_res.value()}; + output_sexp(Sexp().put_props(":info", "add"_sym, + ":path", *path, + ":docid", docid)); + + auto msg_res{store().find_message(docid)}; + if (!msg_res) + throw Error(Error::Code::Store, + "failed to get message at {} (docid={})", *path, docid); + + output(mu_format("(:update {})", + msg_sexp_str(msg_res.value(), docid, {}))); +} + +void +Server::Private::contacts_handler(const Command& cmd) +{ + const auto personal = cmd.boolean_arg(":personal"); + const auto afterstr = cmd.string_arg(":after").value_or(""); + const auto tstampstr = cmd.string_arg(":tstamp").value_or(""); + const auto maxnum = cmd.number_arg(":maxnum").value_or(0 /*unlimited*/); + + const auto after{afterstr.empty() ? 0 : parse_date_time(afterstr, true).value_or(0)}; + const auto tstamp = g_ascii_strtoll(tstampstr.c_str(), NULL, 10); + + mu_debug("find {} contacts last seen >= {:%c} (tstamp: {})", + personal ? "personal" : "any", mu_time(after), tstamp); + + auto match_contact = [&](const Contact& ci)->bool { + if (ci.tstamp < tstamp) + return false; /* already seen? */ + else if (personal && !ci.personal) + return false; /* not personal? */ + else if (ci.message_date < after) + return false; /* too old? */ + else + return true; + }; + + auto n{0}; + auto&& out{make_output_stream()}; + mu_print(out, "("); + store().contacts_cache().for_each([&](const Contact& ci) { + if (!match_contact(ci)) + return true; // continue + mu_println(out.out(), "{}", quote(ci.display_name())); + ++n; + return maxnum == 0 || n < maxnum; + }); + mu_print(out, ")"); + output(mu_format("(:contacts {}\n:tstamp \"{}\")", + out.to_string(), g_get_monotonic_time())); + + mu_debug("sent {} of {} contact(s)", n, store().contacts_cache().size()); +} + +void +Server::Private::data_handler(const Command& cmd) +{ + const auto request_type{unwrap(cmd.symbol_arg(":kind"))}; + + if (request_type == "maildirs") { + auto&& out{make_output_stream()}; + mu_print(out, "("); + for (auto&& mdir: store().maildirs()) + mu_println(out, "{}", quote(std::move(mdir))); + mu_print(out, ")"); + output(mu_format("(:maildirs {})", out.to_string())); + } else + throw Error(Error::Code::InvalidArgument, + "invalid request type '{}'", request_type); +} + + +/* + * creating a message object just to get a path seems a bit excessive maybe + * mu_store_get_path could be added if this turns out to be a problem + */ +static std::string +path_from_docid(const Store& store, Store::Id docid) +{ + auto msg{store.find_message(docid)}; + if (!msg) + throw Error(Error::Code::Store, "could not get message from store"); + + if (auto path{msg->path()}; path.empty()) + throw Error(Error::Code::Store, "could not get path for message {}", + docid); + else + return path; +} + +static std::vector<Store::Id> +determine_docids(const Store& store, const Command& cmd) +{ + auto docid{cmd.number_arg(":docid").value_or(0)}; + const auto msgid{cmd.string_arg(":msgid").value_or("")}; + + if ((docid == 0) == msgid.empty()) + throw Error(Error::Code::InvalidArgument, + "precisely one of docid and msgid must be specified"); + + if (docid != 0) + return {static_cast<Store::Id>(docid)}; + else + return store.find_duplicates(msgid); +} + +size_t +Server::Private::output_results(const QueryResults& qres, size_t batch_size) const +{ + // create an output stream with a file name + size_t n{}, batch_n{}; + auto&& out{make_output_stream()}; + // structured bindings / lambda don't work with some clang. + + mu_print(out, "("); + for (auto&& mi: qres) { + + auto msg{mi.message()}; + if (!msg) + continue; + + auto qm{mi.query_match()}; // construct sexp for a single header. + mu_println(out, "{}", msg_sexp_str(*msg, mi.doc_id(), qm)); + ++n; + ++batch_n; + + if (n % batch_size == 0) { + // batch complete + mu_print(out, ")"); + batch_size = 5000; + output(mu_format("(:headers {})", out.to_string())); + batch_n = 0; + // start a new batch + out = make_output_stream(); + mu_print(out, "("); + } + } + + mu_print(out, ")"); + if (batch_n > 0) + output(mu_format("(:headers {})", out.to_string())); + else + out.unlink(); + + return n; +} + + +void +Server::Private::find_handler(const Command& cmd) +{ + const auto q{cmd.string_arg(":query").value_or("")}; + const auto threads{cmd.boolean_arg(":threads")}; + // perhaps let mu4e set this as frame-lines of the appropriate frame. + const auto batch_size{cmd.number_arg(":batch-size").value_or(200)}; + const auto descending{cmd.boolean_arg(":descending")}; + const auto maxnum{cmd.number_arg(":maxnum").value_or(-1) /*unlimited*/}; + const auto skip_dups{cmd.boolean_arg(":skip-dups")}; + const auto include_related{cmd.boolean_arg(":include-related")}; + + // complicated! + auto sort_field_id = std::invoke([&]()->Field::Id { + if (const auto arg = cmd.symbol_arg(":sortfield"); !arg) + return Field::Id::Date; + else if (arg->length() < 2) + throw Error{Error::Code::InvalidArgument, "invalid sort field '{}'", + *arg}; + else if (const auto field{field_from_name(arg->substr(1))}; !field) + throw Error{Error::Code::InvalidArgument, "invalid sort field '{}'", + *arg}; + else + return field->id; + }); + + if (batch_size < 1) + throw Error{Error::Code::InvalidArgument, "invalid batch-size {}", batch_size}; + + auto qflags{QueryFlags::SkipUnreadable}; // don't show unreadables. + if (descending) + qflags |= QueryFlags::Descending; + if (skip_dups) + qflags |= QueryFlags::SkipDuplicates; + if (include_related) + qflags |= QueryFlags::IncludeRelated; + if (threads) + qflags |= QueryFlags::Threading; + + StopWatch sw{mu_format("{} (indexing: {})", __func__, + indexer().is_running() ? "yes" : "no")}; + + // we need to _lock_ the store while querying (which likely consists of + // multiple actual queries) + grabbing the results. + std::lock_guard l{store_.lock()}; + auto qres{store_.run_query(q, sort_field_id, qflags, maxnum)}; + if (!qres) + throw Error(Error::Code::Query, "failed to run query: {}", qres.error().what()); + + /* before sending new results, send an 'erase' message, so the frontend + * knows it should erase the headers buffer. this will ensure that the + * output of two finds will not be mixed. */ + output_sexp(Sexp().put_props(":erase", Sexp::t_sym)); + const auto bsize{static_cast<size_t>(batch_size)}; + const auto foundnum = output_results(*qres, bsize); + output_sexp(Sexp().put_props(":found", foundnum)); +} + +void +Server::Private::help_handler(const Command& cmd) +{ + const auto command{cmd.symbol_arg(":command").value_or("")}; + const auto full{cmd.bool_arg(":full").value_or(!command.empty())}; + auto&& info_map{command_handler_.info_map()}; + + if (command.empty()) { + mu_println(";; Commands are single-line s-expressions of the form\n" + ";; (<command-name> :param1 val1 :param2 val2 ...)\n" + ";; For instance:\n;; (help :command mkdir)\n" + ";; to get more information about the 'mkdir' command\n;;\n" + ";; The following commands are available:"); + } + + std::vector<std::string> names; + for (auto&& name_cmd: info_map) + names.emplace_back(name_cmd.first); + + std::sort(names.begin(), names.end()); + + for (auto&& name : names) { + const auto& info{info_map.find(name)->second}; + + if (!command.empty() && name != command) + continue; + + mu_println(";; {:<12} -- {}", name, info.docstring); + + if (!full) + continue; + + for (auto&& argname : info.sorted_argnames()) { + const auto& arg{info.args.find(argname)}; + mu_println(";; {:<17} :: {:<24} -- {}", + arg->first, to_string(arg->second), + arg->second.docstring); + } + mu_println(";;"); + } +} + +static Sexp +get_stats(const Indexer::Progress& stats, const std::string& state) +{ + Sexp sexp; + sexp.put_props( + ":info", "index"_sym, + ":status", Sexp::Symbol(state), + ":checked", static_cast<int>(stats.checked), + ":updated", static_cast<int>(stats.updated), + ":cleaned-up", static_cast<int>(stats.removed)); + + return sexp; +} + +void +Server::Private::index_handler(const Command& cmd) +{ + Mu::Indexer::Config conf{}; + conf.cleanup = cmd.boolean_arg(":cleanup"); + conf.lazy_check = cmd.boolean_arg(":lazy-check"); + // ignore .noupdate with an empty store. + conf.ignore_noupdate = store().empty(); + + indexer().stop(); + if (index_thread_.joinable()) + index_thread_.join(); + + // start a background track. + index_thread_ = std::thread([this, conf = std::move(conf)] { + StopWatch sw{"indexing"}; + indexer().start(conf); + while (indexer().is_running()) { + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + output_sexp(get_stats(indexer().progress(), "running"), + Server::OutputFlags::Flush); + } + output_sexp(get_stats(indexer().progress(), "complete"), + Server::OutputFlags::Flush); + }); +} + +void +Server::Private::mkdir_handler(const Command& cmd) +{ + const auto path{cmd.string_arg(":path").value_or("<error>")}; + const auto update{cmd.boolean_arg(":update")}; + + if (path.find(store().root_maildir()) != 0) + throw Error{Error::Code::File, "maildir is not below root-maildir"}; + + if (auto&& res = maildir_mkdir(path, 0755, false); !res) + throw res.error(); + + /* mu4e does a lot of opportunistic 'mkdir', only send it updates when + * requested */ + if (!update) + return; + + output_sexp(Sexp().put_props(":info", "mkdir", + ":message", + mu_format("{} has been created", path))); +} + +void +Server::Private::perform_move(Store::Id docid, + const Message& msg, + const std::string& maildirarg, + Flags flags, + bool new_name, + bool no_view) +{ + bool different_mdir{}; + auto maildir{maildirarg}; + if (maildir.empty()) { + maildir = msg.maildir(); + different_mdir = false; + } else /* are we moving to a different mdir, or is it just flags? */ + different_mdir = maildir != msg.maildir(); + + Store::MoveOptions move_opts{Store::MoveOptions::DupFlags}; + if (new_name) + move_opts |= Store::MoveOptions::ChangeName; + + /* note: we get back _all_ the messages that changed; the first is the + * primary mover; the rest (if present) are any dups affected */ + const auto id_paths{unwrap(store().move_message(docid, maildir, flags, move_opts))}; + for (auto& [id,path]: id_paths) { + auto idmsg{store().find_message(id)}; + if (!idmsg) + throw Error{Error::Code::Xapian, "cannot find message for id {}", id}; + + auto sexpstr = "(:update " + msg_sexp_str(*idmsg, id, {}); + /* note, the :move t thing is a hint to the frontend that it + * could remove the particular header */ + if (different_mdir) + sexpstr += " :move t"; + if (!no_view && id == docid) + sexpstr += " :maybe-view t"; + sexpstr += ')'; + output(std::move(sexpstr)); + } +} + +static Flags +calculate_message_flags(const Message& msg, Option<std::string> flagopt) +{ + const auto flags = std::invoke([&]()->Option<Flags>{ + if (!flagopt) + return msg.flags(); + else + return flags_from_expr(*flagopt, msg.flags()); + }); + + if (!flags) + throw Error{Error::Code::InvalidArgument, + "invalid flags '{}'", flagopt.value_or("")}; + else + return flags.value(); +} + +void +Server::Private::move_docid(Store::Id docid, + Option<std::string> flagopt, + bool new_name, + bool no_view) +{ + if (docid == Store::InvalidId) + throw Error{Error::Code::InvalidArgument, "invalid docid"}; + + auto msg{store_.find_message(docid)}; + if (!msg) + throw Error{Error::Code::Store, "failed to get message from store"}; + + const auto flags = calculate_message_flags(msg.value(), flagopt); + perform_move(docid, *msg, "", flags, new_name, no_view); +} + +/* + * 'move' moves a message to a different maildir and/or changes its flags. + * parameters are *either* a 'docid:' or 'msgid:' pointing to the message, a + * 'maildir:' for the target maildir, and a 'flags:' parameter for the new + * flags. + * + * With :msgid, this is "opportunistic": it's not an error when the given + * message-id does not exist. This is e.g. for the case when tagging possible + * related messages. + */ +void +Server::Private::move_handler(const Command& cmd) +{ + auto maildir{cmd.string_arg(":maildir").value_or("")}; + const auto flagopt{cmd.string_arg(":flags")}; + const auto rename{cmd.boolean_arg(":rename")}; + const auto no_view{cmd.boolean_arg(":noupdate")}; + const auto docids{determine_docids(store_, cmd)}; + + if (docids.empty()) { + if (!!cmd.string_arg(":msgid")) { + // msgid not found: no problem. + mu_debug("no move: '{}' not found", + *cmd.string_arg(":msgid")); + return; + } + // however, if we wanted to be move by msgid, it's worth raising + // an error. + throw Mu::Error{Error::Code::Store, + "message not found in store (docid={})", + cmd.number_arg(":docid").value_or(0)}; + } else if (docids.size() > 1) { + if (!maildir.empty()) // ie. duplicate message-ids. + throw Mu::Error{Error::Code::Store, + "cannot move multiple messages at the same time"}; + // multi. + for (auto&& docid : docids) + move_docid(docid, flagopt, rename, no_view); + return; + } else { + const auto docid{docids.at(0)}; + auto msg = store().find_message(docid) + .or_else([&]{throw Error{Error::Code::InvalidArgument, + "cannot find message {}", docid};}).value(); + + /* if maildir was not specified, take the current one */ + if (maildir.empty()) + maildir = msg.maildir(); + + /* determine the real target flags, which come from the flags-parameter + * we received (ie., flagstr), if any, plus the existing message + * flags. */ + const auto flags = calculate_message_flags(msg, flagopt); + perform_move(docid, msg, maildir, flags, rename, no_view); + } +} + +void +Server::Private::ping_handler(const Command& cmd) +{ + const auto storecount{store().size()}; + if (storecount == (unsigned)-1) + throw Error{Error::Code::Store, "failed to read store"}; + Sexp addrs; + for (auto&& addr : store().config().get<Config::Id::PersonalAddresses>()) + addrs.add(addr); + + output_sexp(Sexp() + .put_props(":pong", "mu") + .put_props(":props", + Sexp().put_props( + ":version", VERSION, + ":personal-addresses", std::move(addrs), + ":database-path", store().path(), + ":root-maildir", store().root_maildir(), + ":doccount", storecount))); +} + +void +Server::Private::queries_handler(const Command& cmd) +{ + const auto queries{cmd.string_vec_arg(":queries") + .value_or(std::vector<std::string>{})}; + + Sexp qresults; + for (auto&& q : queries) { + const auto count{store_.count_query(q)}; + const auto unreadq{mu_format("flag:unread AND ({})", q)}; + const auto unread{store_.count_query(unreadq)}; + qresults.add(Sexp().put_props(":query", q, + ":count", count, + ":unread", unread)); + } + + output_sexp(Sexp(":queries"_sym, std::move(qresults))); +} + + +void +Server::Private::quit_handler(const Command& cmd) +{ + keep_going_ = false; +} + +void +Server::Private::remove_handler(const Command& cmd) +{ + auto docid_opt{cmd.number_arg(":docid")}; + auto path_opt{cmd.string_arg(":path")}; + + if (!!docid_opt == !!path_opt) + throw Error(Error::Code::InvalidArgument, + "must pass precisely one of :docid and :path"); + std::string path; + Store::Id docid{}; + if (docid = docid_opt.value_or(0); docid != 0) + path = path_from_docid(store(), docid); + else + path = path_opt.value(); + + if (::unlink(path.c_str()) != 0 && errno != ENOENT) + throw Error(Error::Code::File, + "could not delete {}: {}", path, g_strerror(errno)); + + if (!store().remove_message(path)) + mu_warning("failed to remove message @ {} ({}) from store", path, docid); + else + mu_debug("removed message @ {} @ ({})", path, docid); + + output_sexp(Sexp().put_props(":remove", docid)); // act as if it worked. +} + +void +Server::Private::view_mark_as_read(Store::Id docid, Message&& msg, bool rename) +{ + auto new_flags = [](const Message& m)->Option<Flags> { + auto nflags = flags_from_delta_expr("+S-u-N", m.flags()); + if (!nflags || nflags == m.flags()) + return Nothing; // nothing to do + else + return nflags; + }; + + auto&& nflags = new_flags(msg); + if (!nflags) { // nothing to move, just send the message for viewing. + output(mu_format("(:view {})", msg_sexp_str(msg, docid, {}))); + return; + } + + // move message + dups, present results. + Store::MoveOptions move_opts{Store::MoveOptions::DupFlags}; + if (rename) + move_opts |= Store::MoveOptions::ChangeName; + + const auto ids{Store::id_vec(unwrap(store().move_message(docid, {}, nflags, move_opts)))}; + for (auto&& [id, moved_msg]: store().find_messages(ids)) + output(mu_format("({} {})", id == docid ? ":view" : ":update", + msg_sexp_str(moved_msg, id, {}))); +} + +void +Server::Private::view_handler(const Command& cmd) +{ + StopWatch sw{mu_format("{} (indexing: {})", __func__, indexer().is_running())}; + + const auto mark_as_read{cmd.boolean_arg(":mark-as-read")}; + /* for now, do _not_ rename, as it seems to confuse mbsync */ + const auto rename{false}; + //const auto rename{get_bool_or(params, ":rename")}; + + const auto docids{determine_docids(store(), cmd)}; + + if (docids.empty()) + throw Error{Error::Code::Store, "failed to find message for view"}; + const auto docid{docids.at(0)}; + auto msg = store().find_message(docid) + .or_else([]{throw Error{Error::Code::Store, + "failed to find message for view"};}).value(); + + /* if the message should not be marked-as-read, we're done. */ + if (!mark_as_read) + output(mu_format("(:view {})", msg_sexp_str(msg, docid, {}))); + else + view_mark_as_read(docid, std::move(msg), rename); + /* otherwise, mark message and and possible dups as read */ +} + +Server::Server(Store& store, const Server::Options& opts, Server::Output output) + : priv_{std::make_unique<Private>(store, opts, output)} +{} + +Server::~Server() = default; + +bool +Server::invoke(const std::string& expr) noexcept +{ + return priv_->invoke(expr); +} + +/* LCOV_EXCL_STOP */ diff --git a/lib/mu-server.hh b/lib/mu-server.hh new file mode 100644 index 0000000..0ceaa68 --- /dev/null +++ b/lib/mu-server.hh @@ -0,0 +1,89 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_SERVER_HH__ +#define MU_SERVER_HH__ + +#include <memory> +#include <functional> + +#include <utils/mu-sexp.hh> +#include <utils/mu-utils.hh> +#include <mu-store.hh> + +/* LCOV_EXCL_START */ + +namespace Mu { + +/** + * @brief Implements the mu server, as used by mu4e. + */ +class Server { +public: + enum struct OutputFlags { + None = 0, + Flush = 1 << 0, /**< flush output buffer after */ + }; + + /** + * Prototype for output function + * + * @param str a string + * @param flags flags that influence the behavior + */ + using Output = std::function<void(const std::string& str, OutputFlags flags)>; + + struct Options { + bool allow_temp_file; /**< temp file optimization allowed? */ + }; + + /** + * Construct a new server + * + * @param store a message store object + * @param output callable for the server responses. + */ + Server(Store& store, const Options& opts, Output output); + + /** + * DTOR + */ + ~Server(); + + /** + * Invoke a call on the server. + * + * @param expr the s-expression to call + * + * @return true if we the server is still ready for more + * calls, false when it should quit. + */ + bool invoke(const std::string& expr) noexcept; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; +MU_ENABLE_BITOPS(Server::OutputFlags); + +} // namespace Mu + +/* LCOV_EXCL_STOP */ + +#endif /* MU_SERVER_HH__ */ diff --git a/lib/mu-store.cc b/lib/mu-store.cc new file mode 100644 index 0000000..eb08eac --- /dev/null +++ b/lib/mu-store.cc @@ -0,0 +1,673 @@ +/* +** Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-store.hh" + +#include <chrono> +#include <mutex> +#include <array> +#include <cstdlib> +#include <stdexcept> +#include <unordered_map> +#include <atomic> +#include <type_traits> +#include <iostream> +#include <cstring> + +#include "mu-maildir.hh" +#include "mu-query.hh" +#include "mu-xapian-db.hh" +#include "mu-scanner.hh" + +#include "utils/mu-error.hh" + +#include "utils/mu-utils.hh" +#include <utils/mu-utils-file.hh> + +using namespace Mu; + +static_assert(std::is_same<Store::Id, Xapian::docid>::value, "wrong type for Store::Id"); + +// Properties +constexpr auto ExpectedSchemaVersion = MU_STORE_SCHEMA_VERSION; + +static std::string +remove_slash(const std::string& str) +{ + auto clean{str}; + while (!clean.empty() && clean[clean.length() - 1] == '/') + clean.pop_back(); + + return clean; +} + +struct Store::Private { + + Private(const std::string& path, bool readonly): + xapian_db_{XapianDb(path, readonly ? XapianDb::Flavor::ReadOnly + : XapianDb::Flavor::Open)}, + config_{xapian_db_}, + contacts_cache_{config_}, + root_maildir_{remove_slash(config_.get<Config::Id::RootMaildir>())}, + message_opts_{make_message_options(config_)} + {} + + Private(const std::string& path, const std::string& root_maildir, + Option<const Config&> conf): + xapian_db_{XapianDb(path, XapianDb::Flavor::CreateOverwrite)}, + config_{make_config(xapian_db_, root_maildir, conf)}, + contacts_cache_{config_}, + root_maildir_{remove_slash(config_.get<Config::Id::RootMaildir>())}, + message_opts_{make_message_options(config_)} + {} + + ~Private() try { + mu_debug("closing store @ {}", xapian_db_.path()); + if (!xapian_db_.read_only()) + contacts_cache_.serialize(); + } catch (...) { + mu_critical("caught exception in store dtor"); + } + + Config make_config(XapianDb& xapian_db, const std::string& root_maildir, + Option<const Config&> conf) { + + if (!g_path_is_absolute(root_maildir.c_str())) + throw Error{Error::Code::File, + "root maildir path is not absolute ({})", + root_maildir}; + + Config config{xapian_db}; + if (conf) + config.import_configurable(*conf); + + config.set<Config::Id::RootMaildir>(remove_slash(root_maildir)); + config.set<Config::Id::SchemaVersion>(ExpectedSchemaVersion); + + return config; + } + + Message::Options make_message_options(const Config& conf) { + if (conf.get<Config::Id::SupportNgrams>()) + return Message::Options::SupportNgrams; + else + return Message::Options::None; + } + + Option<Message> find_message_unlocked(Store::Id docid) const; + Store::IdVec find_duplicates_unlocked(const Store& store, + const std::string& message_id) const; + + Result<Store::Id> add_message_unlocked(Message& msg); + Result<Store::Id> update_message_unlocked(Message& msg, Store::Id docid); + Result<Store::Id> update_message_unlocked(Message& msg, const std::string& old_path); + + + using PathMessage = std::pair<std::string, Message>; + Result<PathMessage> move_message_unlocked(Message&& msg, + Option<const std::string&> target_mdir, + Option<Flags> new_flags, + MoveOptions opts); + XapianDb xapian_db_; + Config config_; + ContactsCache contacts_cache_; + std::unique_ptr<Indexer> indexer_; + + const std::string root_maildir_; + const Message::Options message_opts_; + + std::mutex lock_; +}; + + +Result<Store::Id> +Store::Private::add_message_unlocked(Message& msg) +{ + auto&& docid{xapian_db_.add_document(msg.document().xapian_document())}; + if (docid) + mu_debug("added message @ {}; docid = {}", msg.path(), *docid); + + return docid; +} + + +Result<Store::Id> +Store::Private::update_message_unlocked(Message& msg, Store::Id docid) +{ + auto&& res{xapian_db_.replace_document(docid, msg.document().xapian_document())}; + if (res) + mu_debug("updated message @ {}; docid = {}", msg.path(), *res); + + return res; +} + +Result<Store::Id> +Store::Private::update_message_unlocked(Message& msg, const std::string& path_to_replace) +{ + return xapian_db_.replace_document( + field_from_id(Field::Id::Path).xapian_term(path_to_replace), + msg.document().xapian_document()); +} + +Option<Message> +Store::Private::find_message_unlocked(Store::Id docid) const +{ + if (auto&& doc{xapian_db_.document(docid)}; !doc) + return Nothing; + else if (auto&& msg{Message::make_from_document(std::move(*doc))}; !msg) + return Nothing; + else + return Some(std::move(*msg)); +} + +Store::IdVec +Store::Private::find_duplicates_unlocked(const Store& store, + const std::string& message_id) const +{ + if (message_id.empty() || message_id.size() > MaxTermLength) { + mu_warning("invalid message-id '{}'", message_id); + return {}; + } + + auto expr{mu_format("{}:{}", + field_from_id(Field::Id::MessageId).shortcut, + message_id)}; + if (auto&& res{store.run_query(expr)}; !res) { + mu_warning("error finding message-ids: {}", res.error().what()); + return {}; + + } else { + Store::IdVec ids; + ids.reserve(res->size()); + for (auto&& mi: *res) + ids.emplace_back(mi.doc_id()); + return ids; + } +} + + +Store::Store(const std::string& path, Store::Options opts) + : priv_{std::make_unique<Private>(path, none_of(opts & Store::Options::Writable))} +{ + if (none_of(opts & Store::Options::Writable) && + any_of(opts & Store::Options::ReInit)) + throw Mu::Error(Error::Code::InvalidArgument, + "Options::ReInit requires Options::Writable"); + + const auto s_version{config().get<Config::Id::SchemaVersion>()}; + if (any_of(opts & Store::Options::ReInit)) { + /* don't try to recover from version with an incompatible scheme */ + if (s_version < 500) + throw Mu::Error(Error::Code::CannotReinit, + "old schema ({}) is too old to re-initialize from", + s_version).add_hint("Invoke 'mu init' without '--reinit'; " + "see mu-init(1) for details"); + const auto old_root_maildir{root_maildir()}; + + MemDb mem_db; + Config old_config(mem_db); + old_config.import_configurable(config()); + + this->priv_.reset(); + /* and create a new one "in place" */ + Store new_store(path, old_root_maildir, old_config); + this->priv_ = std::move(new_store.priv_); + } + + /* otherwise, the schema version should match. */ + if (s_version != ExpectedSchemaVersion) + throw Mu::Error(Error::Code::SchemaMismatch, + "expected schema-version {}, but got {}", + ExpectedSchemaVersion, s_version). + add_hint("Please (re)initialize with 'mu init'; see mu-init(1) for details"); +} + +Store::Store(const std::string& path, + const std::string& root_maildir, + Option<const Config&> conf): + priv_{std::make_unique<Private>(path, root_maildir, conf)} +{} + +Store::Store(Store&& other) +{ + priv_ = std::move(other.priv_); + priv_->indexer_.reset(); +} + +Store::~Store() = default; + +Store::Statistics +Store::statistics() const +{ + Statistics stats{}; + + stats.size = size(); + stats.last_change = config().get<Config::Id::LastChange>(); + stats.last_index = config().get<Config::Id::LastIndex>(); + + return stats; +} + +const XapianDb& +Store::xapian_db() const +{ + return priv_->xapian_db_; +} + +XapianDb& +Store::xapian_db() +{ + return priv_->xapian_db_; +} + +const Config& +Store::config() const +{ + return priv_->config_; +} + +Config& +Store::config() +{ + return priv_->config_; +} + +const std::string& +Store::root_maildir() const { + return priv_->root_maildir_; +} + +const ContactsCache& +Store::contacts_cache() const +{ + return priv_->contacts_cache_; +} + +Indexer& +Store::indexer() +{ + std::lock_guard guard{priv_->lock_}; + + if (xapian_db().read_only()) + throw Error{Error::Code::Store, "no indexer for read-only store"}; + else if (!priv_->indexer_) + priv_->indexer_ = std::make_unique<Indexer>(*this); + + return *priv_->indexer_.get(); +} + +Result<Store::Id> +Store::add_message(Message& msg, bool is_new) +{ + const auto mdir{maildir_from_path(msg.path(), root_maildir())}; + if (!mdir) + return Err(mdir.error()); + if (auto&& res = msg.set_maildir(mdir.value()); !res) + return Err(res.error()); + + // we shouldn't mix ngrams/non-ngrams messages. + if (any_of(msg.options() & Message::Options::SupportNgrams) != + any_of(message_options() & Message::Options::SupportNgrams)) + return Err(Error::Code::InvalidArgument, "incompatible message options"); + + /* add contacts from this message to cache; this cache + * also determines whether those contacts are _personal_, i.e. match + * our personal addresses. + * + * if a message has any personal contacts, mark it as personal; do + * this by updating the message flags. + */ + bool is_personal{}; + priv_->contacts_cache_.add(msg.all_contacts(), is_personal); + if (is_personal) + msg.set_flags(msg.flags() | Flags::Personal); + + std::lock_guard guard{priv_->lock_}; + auto&& res = is_new ? + priv_->add_message_unlocked(msg) : + priv_->update_message_unlocked(msg, msg.path()); + if (!res) + return Err(res.error()); + + mu_debug("added {}{}message @ {}; docid = {}", + is_new ? "new " : "", is_personal ? "personal " : "", msg.path(), *res); + + return res; +} + +Result<Store::Id> +Store::add_message(const std::string& path, bool is_new) +{ + if (auto msg{Message::make_from_path(path, priv_->message_opts_)}; !msg) + return Err(msg.error()); + else + return add_message(msg.value(), is_new); +} + + +bool +Store::remove_message(const std::string& path) +{ + const auto term{field_from_id(Field::Id::Path).xapian_term(path)}; + + std::lock_guard guard{priv_->lock_}; + + xapian_db().delete_document(term); + mu_debug("deleted message @ {} from store", path); + return true; +} + +void +Store::remove_messages(const std::vector<Store::Id>& ids) +{ + std::lock_guard guard{priv_->lock_}; + + XapianDb::Transaction tx (xapian_db()); // RAII + + for (auto&& id : ids) + xapian_db().delete_document(id); +} + + +Option<Message> +Store::find_message(Store::Id docid) const +{ + std::lock_guard guard{priv_->lock_}; + + return priv_->find_message_unlocked(docid); +} + +Option<Store::Id> +Store::find_message_id(const std::string& path) const +{ + constexpr auto path_field{field_from_id(Field::Id::Path)}; + + std::lock_guard guard{priv_->lock_}; + + auto enq{xapian_db().enquire()}; + enq.set_query(Xapian::Query{path_field.xapian_term(path)}); + + if (auto mset{enq.get_mset(0, 1)}; mset.empty()) + return Nothing; // message not found + else + return Some(*mset.begin()); +} + + +Store::IdMessageVec +Store::find_messages(IdVec ids) const +{ + std::lock_guard guard{priv_->lock_}; + + IdMessageVec id_msgs; + for (auto&& id: ids) { + if (auto&& msg{priv_->find_message_unlocked(id)}; msg) + id_msgs.emplace_back(std::make_pair(id, std::move(*msg))); + } + + return id_msgs; +} + +/** + * Move a message in store and filesystem; with DryRun, only calculate the target name. + * + * Lock is assumed taken already + * + * @param id message id + * @param target_mdir target_mdir (or Nothing for current) + * @param new_flags new flags (or Nothing) + * @param opts move_options + * + * @return the Message after the moving, or an Error + */ +Result<Store::Private::PathMessage> +Store::Private::move_message_unlocked(Message&& msg, + Option<const std::string&> target_mdir, + Option<Flags> new_flags, + MoveOptions opts) +{ + const auto old_path = msg.path(); + const auto target_flags = new_flags.value_or(msg.flags()); + const auto target_maildir = target_mdir.value_or(msg.maildir()); + + /* 1. first determine the file system path of the target */ + const auto target_path = + maildir_determine_target(msg.path(), root_maildir_, + target_maildir, target_flags, + any_of(opts & MoveOptions::ChangeName)); + if (!target_path) + return Err(target_path.error()); + + // in dry-run mode, we only determine the target-path + if (none_of(opts & MoveOptions::DryRun)) { + + /* 2. let's move it */ + if (const auto res = maildir_move_message(msg.path(), target_path.value()); !res) + return Err(res.error()); + + /* 3. file move worked, now update the message with the new info.*/ + if (auto&& res = msg.update_after_move( + target_path.value(), target_maildir, target_flags); !res) + return Err(res.error()); + + /* 4. update message worked; re-store it */ + if (auto&& res = update_message_unlocked(msg, old_path); !res) + return Err(res.error()); + } + + /* 6. Profit! */ + return Ok(PathMessage{std::move(*target_path), std::move(msg)}); +} + +Store::IdVec +Store::find_duplicates(const std::string& message_id) const +{ + std::lock_guard guard{priv_->lock_}; + + return priv_->find_duplicates_unlocked(*this, message_id); +} + + +Result<Store::IdPathVec> +Store::move_message(Store::Id id, + Option<const std::string&> target_mdir, + Option<Flags> new_flags, + MoveOptions opts) +{ + auto filter_dup_flags=[](Flags old_flags, Flags new_flags) -> Flags { + new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Draft); + new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Flagged); + new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Trashed); + return new_flags; + }; + + std::lock_guard guard{priv_->lock_}; + + auto msg{priv_->find_message_unlocked(id)}; + if (!msg) + return Err(Error::Code::Store, "cannot find message <{}>", id); + + const auto message_id{msg->message_id()}; + auto res{priv_->move_message_unlocked(std::move(*msg), target_mdir, new_flags, opts)}; + if (!res) + return Err(res.error()); + + IdPathVec id_paths{{id, res->first}}; + if (none_of(opts & Store::MoveOptions::DupFlags) || message_id.empty() || !new_flags) + return Ok(std::move(id_paths)); + + /* handle the dup-flags case; i.e. apply (a subset of) the flags to + * all messages with the same message-id as well */ + auto dups{priv_->find_duplicates_unlocked(*this, message_id)}; + for (auto&& dupid: dups) { + + if (dupid == id) + continue; // already + + auto dup_msg{priv_->find_message_unlocked(dupid)}; + if (!dup_msg) + continue; // no such message + + /* For now, don't change Draft/Flagged/Trashed */ + const auto dup_flags{filter_dup_flags(dup_msg->flags(), *new_flags)}; + /* use the updated new_flags and MoveOptions without DupFlags (so we don't + * recurse) */ + opts = opts & ~MoveOptions::DupFlags; + if (auto dup_res = priv_->move_message_unlocked( + std::move(*dup_msg), Nothing, dup_flags, opts); !dup_res) + mu_warning("failed to move dup: {}", dup_res.error().what()); + else + id_paths.emplace_back(dupid, dup_res->first); + } + + // sort the dup paths by name; + std::sort(id_paths.begin() + 1, id_paths.end(), + [](const auto& idp1, const auto& idp2) { return idp1.second < idp2.second; }); + + return Ok(std::move(id_paths)); +} + +Store::IdVec +Store::id_vec(const IdPathVec& ips) +{ + IdVec idv; + for (auto&& ip: ips) + idv.emplace_back(ip.first); + + return idv; +} + + +time_t +Store::dirstamp(const std::string& path) const +{ + std::string ts; + + { + std::unique_lock lock{priv_->lock_}; + ts = xapian_db().metadata(path); + } + + return ts.empty() ? 0 /*epoch*/ : ::strtoll(ts.c_str(), {}, 16); +} + +void +Store::set_dirstamp(const std::string& path, time_t tstamp) +{ + std::unique_lock lock{priv_->lock_}; + + xapian_db().set_metadata(path, mu_format("{:x}", tstamp)); +} + +bool +Store::contains_message(const std::string& path) const +{ + std::unique_lock lock{priv_->lock_}; + + return xapian_db().term_exists(field_from_id(Field::Id::Path).xapian_term(path)); +} + +std::size_t +Store::for_each_message_path(Store::ForEachMessageFunc msg_func) const +{ + size_t n{}; + + xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + auto enq{xapian_db().enquire()}; + + enq.set_query(Xapian::Query::MatchAll); + enq.set_cutoff(0, 0); + + Xapian::MSet matches(enq.get_mset(0, xapian_db().size())); + constexpr auto path_no{field_from_id(Field::Id::Path).value_no()}; + for (auto&& it = matches.begin(); it != matches.end(); ++it, ++n) + if (!msg_func(*it, it.get_document().get_value(path_no))) + break; + }); + + return n; +} + +std::size_t +Store::for_each_term(Field::Id field_id, Store::ForEachTermFunc func) const +{ + return xapian_db().all_terms(field_from_id(field_id).xapian_term(), func); +} + +std::mutex& +Store::lock() const +{ + return priv_->lock_; +} + +Result<QueryResults> +Store::run_query(const std::string& expr, + Field::Id sortfield_id, + QueryFlags flags, size_t maxnum) const +{ + return Query{*this}.run(expr, sortfield_id, flags, maxnum); +} + +size_t +Store::count_query(const std::string& expr) const +{ + return xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + Query q{*this}; + return q.count(expr); }, 0); +} + +std::string +Store::parse_query(const std::string& expr, bool xapian) const +{ + return xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + Query q{*this}; + + return q.parse(expr, xapian); + }, std::string{}); +} + + +std::vector<std::string> +Store::maildirs() const +{ + std::vector<std::string> mdirs; + const auto prefix_size{root_maildir().size()}; + + Scanner::Handler handler = [&](const std::string& path, auto&& _1, auto&& _2) { + auto md{path.substr(prefix_size)}; + mdirs.emplace_back(md.empty() ? "/" : std::move(md)); + return true; + }; + + Scanner scanner{root_maildir(), handler, Scanner::Mode::MaildirsOnly}; + scanner.start(); + std::sort(mdirs.begin(), mdirs.end()); + + return mdirs; +} + +Message::Options +Store::message_options() const +{ + return priv_->message_opts_; +} diff --git a/lib/mu-store.hh b/lib/mu-store.hh new file mode 100644 index 0000000..dd49045 --- /dev/null +++ b/lib/mu-store.hh @@ -0,0 +1,490 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_STORE_HH__ +#define MU_STORE_HH__ + +#include <string> +#include <vector> +#include <mutex> +#include <ctime> +#include <memory> + +#include "mu-contacts-cache.hh" +#include "mu-xapian-db.hh" +#include "mu-config.hh" +#include "mu-indexer.hh" +#include "mu-query-results.hh" + +#include <utils/mu-utils.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> + +#include <message/mu-message.hh> + +namespace Mu { + +class Store { +public: + using Id = Xapian::docid; /**< Id for a message in the store */ + static constexpr Id InvalidId = 0; /**< Invalid store id */ + using IdVec = std::vector<Id>; /**< Vector of document ids */ + using IdPathVec = std::vector<std::pair<Id, std::string>>; + /**< vector of id, path pairs */ + + /** + * Configuration options. + */ + enum struct Options { + None = 0, /**< No specific options */ + Writable = 1 << 0, /**< Open in writable mode */ + ReInit = 1 << 1, /**< Re-initialize based on existing */ + }; + + /** + * Make a store for an existing document database + * + * @param path path to the database + * @param options startup options + * + * @return A store or an error. + */ + static Result<Store> make(const std::string& path, + Options opts=Options::None) noexcept { + return xapian_try_result( + [&]{return Ok(Store{path, opts});}); + } + + /** + * Construct a store for a not-yet-existing document database + * + * @param path path to the database + * @param root_maildir absolute path to maildir to use for this store + * @param conf a configuration object + * + * @return a store or an error + */ + static Result<Store> make_new(const std::string& path, + const std::string& root_maildir, + Option<const Config&> conf={}) noexcept { + return xapian_try_result( + [&]{return Ok(Store(path, root_maildir, conf));}); + } + + /** + * Move CTOR + * + */ + Store(Store&&); + + /** + * DTOR + */ + ~Store(); + + /** + * Store statistics. Unlike the properties, these can change + * during the lifetime of a store. + * + */ + struct Statistics { + size_t size; /**< number of messages in store */ + ::time_t last_change; /**< last time any update happened */ + ::time_t last_index; /**< last time an indexing op was performed */ + }; + + /** + * Get store statistics + * + * @return statistics + */ + Statistics statistics() const; + + /** + * Get the underlying xapian db object + * + * @return the XapianDb for this store + */ + const XapianDb& xapian_db() const; + XapianDb& xapian_db(); + + /** + * Get the Config for this store + * + * @return the Config + */ + const Config& config() const; + Config& config(); + + /** + * Get the ContactsCache object for this store + * + * @return the Contacts object + */ + const ContactsCache& contacts_cache() const; + + /** + * Get the Indexer associated with this store. It is an error to call + * this on a read-only store. + * + * @return the indexer. + */ + Indexer& indexer(); + + /** + * Run a query; see the `mu-query` man page for the syntax. + * + * Multi-threaded callers must acquire the lock and keep it + * at least as long as the return value. + * + * @param expr the search expression + * @param sortfieldid the sortfield-id. If the field is NONE, sort by DATE + * @param flags query flags + * @param maxnum maximum number of results to return. 0 for 'no limit' + * + * @return the query-results or an error. + */ + std::mutex& lock() const; + Result<QueryResults> run_query(const std::string& expr, + Field::Id sortfield_id = Field::Id::Date, + QueryFlags flags = QueryFlags::None, + size_t maxnum = 0) const; + + /** + * run a Xapian query merely to count the number of matches; for the + * syntax, please refer to the mu-query manpage + * + * @param expr the search expression; use "" to match all messages + * + * @return the number of matches + */ + size_t count_query(const std::string& expr = "") const; + + /** + * For debugging, get the internal string representation of the parsed + * query + * + * @param expr a xapian search expression + * @param xapian if true, show Xapian's internal representation, + * otherwise, mu's. + * + * @return the string representation of the query + */ + std::string parse_query(const std::string& expr, bool xapian) const; + + /** + * Add or update a message to the store. When planning to write many + * messages, it's much faster to do so in a transaction. If so, set + * @param in_transaction to true. When done with adding messages, call + * commit(). + * + * Optimization: If you are sure the message (i.e., a message with the + * given file-system path) does not yet exist in the database, ie., when + * doing the initial indexing, set @p is_new to true since we then don't + * have to check for the existing message. + * + * @param msg a message + * @param is_new whether this is a completely new message + * + * @return the doc id of the added message or an error. + */ + Result<Id> add_message(Message& msg, bool is_new = false); + Result<Id> add_message(const std::string& path, bool is_new = false); + + /** + * Remove a message from the store. It will _not_ remove the message + * from the file system. + * + * @param path the message path. + * + * @return true if removing happened; false otherwise. + */ + bool remove_message(const std::string& path); + + /** + * Remove a number if messages from the store. It will _not_ remove the + * message from the file system. + * + * @param ids vector with store ids for the message + */ + void remove_messages(const std::vector<Id>& ids); + + /** + * Remove a message from the store. It will _not_ remove the message + * from the file system. + * + * @param id the store id for the message + */ + void remove_message(Id id) { remove_messages({id}); } + + /** + * Find message in the store. + * + * @param id doc id for the message to find + * + * @return a message (if found) or Nothing + */ + Option<Message> find_message(Id id) const; + + /** + * Find a message's docid based on its path + * + * @param path path to the message + * + * @return the docid or Nothing if not found + */ + Option<Id> find_message_id(const std::string& path) const; + + /** + * Find the messages for the given ids + * + * @param ids document ids for the message + * + * @return id, message pairs for the messages found + * (which not necessarily _all_ of the ids) + */ + using IdMessageVec = std::vector<std::pair<Id, Message>>; + IdMessageVec find_messages(IdVec ids) const; + + /** + * Find the ids for all messages with a give message-id + * + * @param message_id a message id + * + * @return the ids of all messages with the given message-id + */ + IdVec find_duplicates(const std::string& message_id) const; + + /** + * does a certain message exist in the store already? + * + * @param path the message path + * + * @return true if the message exists in the store, false otherwise + */ + bool contains_message(const std::string& path) const; + + /** + * Options for moving + * + */ + enum struct MoveOptions { + None = 0, /**< Defaults */ + ChangeName = 1 << 0, /**< Change the name when moving */ + DupFlags = 1 << 1, /**< Update flags for duplicate messages too */ + DryRun = 1 << 2, /**< Don't really move, just determine target paths */ + }; + + /** + * Move a message both in the filesystem and in the store. After a successful move, the + * message is updated. + * + * @param id the id for some message + * @param target_mdir the target maildir (if any) + * @param new_flags new flags (if any) + * @param opts move options + * + * @return Result, either an IdPathVec with ids and paths for the moved message(s) or some + * error. Note that in case of success at least one message is returned, and only with + * MoveOptions::DupFlags can it be more than one. + * + * The first element of the IdPathVec, is the main message that got move; any subsequent + * (if any) are the duplicate paths, sorted by path-name. + */ + Result<IdPathVec> move_message(Store::Id id, + Option<const std::string&> target_mdir = Nothing, + Option<Flags> new_flags = Nothing, + MoveOptions opts = MoveOptions::None); + /** + * Convert IdPathVec -> IdVec + * + * @param ips idpath vector + * + * @return vector of ids + */ + static IdVec id_vec(const IdPathVec& ips); + + /** + * Prototype for the ForEachMessageFunc + * + * @param id :t store Id for the message + * @param path: the absolute path to the message + * + * @return true if for_each should continue; false to quit + */ + using ForEachMessageFunc = std::function<bool(Id, const std::string&)>; + + /** + * Call @param func for each document in the store. This takes a lock on + * the store, so the func should _not_ call any other Store:: methods. + * + * @param func a Callable invoked for each message. + * + * @return the number of times func was invoked + */ + size_t for_each_message_path(ForEachMessageFunc func) const; + + /** + * Prototype for the ForEachTermFunc + * + * @param term: + * + * @return true if for_each should continue; false to quit + */ + using ForEachTermFunc = std::function<bool(const std::string&)>; + + /** + * Call @param func for each term for the given field in the store. This + * takes a lock on the store, so the func should _not_ call any other + * Store:: methods. + * + * @param id the field id + * @param func a Callable invoked for each message. + * + * @return the number of times func was invoked + */ + size_t for_each_term(Field::Id id, ForEachTermFunc func) const; + + /** + * Get the timestamp for some message, or 0 if not found + * + * @param path the path + * + * @return the timestamp, or 0 if not found + */ + time_t message_tstamp(const std::string& path) const; + + /** + * Get the timestamp for some directory + * + * @param path the path + * + * @return the timestamp, or 0 if not found + */ + time_t dirstamp(const std::string& path) const; + + /** + * Set the timestamp for some directory + * + * @param path a filesystem path + * @param tstamp the timestamp for that path + */ + void set_dirstamp(const std::string& path, time_t tstamp); + + /* + * + * Some convenience + * + */ + + /** + * Get the Xapian database-path for this store + * + * @return the path + */ + const std::string& path() const { return xapian_db().path(); } + + /** + * Get the root-maildir for this store + * + * @return the root-maildir + */ + const std::string& root_maildir() const; + + /** + * Get the number of messages in the store + * + * @return the number + */ + size_t size() const { return xapian_db().size(); } + + /** + * Is the store empty? + * + * @return true or false + */ + bool empty() const { return xapian_db().empty(); } + + + /** + * Get the list of maildirs, that is, the list of maildirs + * under root_maildir, without file-system prefix. + * + * This does a file-system scan. + * + * @return list of maildirs + */ + std::vector<std::string> maildirs() const; + + + /** + * Compatible message-options for this store + * + * @return message-options. + */ + Message::Options message_options() const; + + + /* + * _almost_ private + */ + + /** + * Get a reference to the private data. For internal use. + * + * @return private reference. + */ + struct Private; + std::unique_ptr<Private>& priv() { return priv_; } + const std::unique_ptr<Private>& priv() const { return priv_; } + +private: + /** + * Construct a store for an existing document database + * + * @param path path to the database + * @param options startup options + */ + Store(const std::string& path, Options opts=Options::None); + + /** + * Construct a store for a not-yet-existing document database + * + * @param path path to the database + * @param config a configuration object + */ + Store(const std::string& path, const std::string& root_maildir, + Option<const Config&> conf); + + std::unique_ptr<Private> priv_; +}; + +MU_ENABLE_BITOPS(Store::Options); +MU_ENABLE_BITOPS(Store::MoveOptions); + +static inline std::string +format_as(const Store& store) +{ + return mu_format("store ({}/{})", format_as(store.xapian_db()), + store.root_maildir()); +} + +} // namespace Mu + +#endif /* MU_STORE_HH__ */ diff --git a/lib/mu-xapian-db.cc b/lib/mu-xapian-db.cc new file mode 100644 index 0000000..a8c897a --- /dev/null +++ b/lib/mu-xapian-db.cc @@ -0,0 +1,148 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#include "mu-xapian-db.hh" +#include "utils/mu-utils.hh" +#include <inttypes.h> +#include <mu-config.hh> + +#include <mutex> + +using namespace Mu; + +const Xapian::Database& +XapianDb::db() const +{ + if (std::holds_alternative<Xapian::WritableDatabase>(db_)) + return std::get<Xapian::WritableDatabase>(db_); + else + return std::get<Xapian::Database>(db_); +} + +Xapian::WritableDatabase& +XapianDb::wdb() +{ + if (read_only()) + throw std::runtime_error("database is read-only"); + return std::get<Xapian::WritableDatabase>(db_); +} + +bool +XapianDb::read_only() const +{ + return !std::holds_alternative<Xapian::WritableDatabase>(db_); +} + +const std::string& +XapianDb::path() const +{ + return path_; +} + +void +XapianDb::set_timestamp(const std::string_view key) +{ + wdb().set_metadata(std::string{key}, mu_format("{}", ::time({}))); +} + +using Flavor = XapianDb::Flavor; + +static std::string +make_path(const std::string& db_path, Flavor flavor) +{ + if (flavor != Flavor::ReadOnly) { + /* we do our own flushing, set Xapian's internal one as + * the backstop*/ + g_setenv("XAPIAN_FLUSH_THRESHOLD", "500000", 1); + /* create path if needed */ + if (g_mkdir_with_parents(db_path.c_str(), 0700) != 0) + throw Error(Error::Code::File, "failed to create database dir {}: {}", + db_path, ::strerror(errno)); + } + + return db_path; +} + +static XapianDb::DbType +make_db(const std::string& db_path, Flavor flavor) +{ + switch (flavor) { + + case Flavor::ReadOnly: + return Xapian::Database(db_path); + case Flavor::Open: + return Xapian::WritableDatabase(db_path, Xapian::DB_OPEN); + case Flavor::CreateOverwrite: + return Xapian::WritableDatabase(db_path, Xapian::DB_CREATE_OR_OVERWRITE); + /* LCOV_EXCL_START*/ + default: + throw std::logic_error("unknown flavor"); + /* LCOV_EXCL_STOP*/ + } +} + +XapianDb::XapianDb(const std::string& db_path, Flavor flavor): + path_(make_path(db_path, flavor)), + db_(make_db(path_, flavor)), + batch_size_{Config(*this).get<Config::Id::BatchSize>()} +{ + if (flavor == Flavor::CreateOverwrite) + set_timestamp(MetadataIface::created_key); + + mu_debug("created {} / {} (batch-size: {})", flavor, *this, batch_size_); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" +#include "config.h" +#include "mu-store.hh" + +static void +test_errors() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + + XapianDb xdb(tdir.path(), Flavor::ReadOnly); + g_assert_true(xdb.read_only()); + + g_assert_false(!!xdb.delete_document("Boo")); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/xapian-db/errors", test_errors); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-xapian-db.hh b/lib/mu-xapian-db.hh new file mode 100644 index 0000000..f9753c6 --- /dev/null +++ b/lib/mu-xapian-db.hh @@ -0,0 +1,575 @@ +/* +** Copyright (C) 2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_XAPIAN_DB_HH__ +#define MU_XAPIAN_DB_HH__ + +#include <variant> +#include <memory> +#include <string> +#include <mutex> +#include <functional> +#include <unordered_map> + +#include <glib.h> + +#include <utils/mu-result.hh> +#include <utils/mu-utils.hh> + +/* starting with 1.4.6, Xapian supports C++ move semantics, + * but only with XAPIAN_MOVE_SEMANTICS defined + */ +#ifndef XAPIAN_MOVE_SEMANTICS +#define XAPIAN_MOVE_SEMANTICS +#endif /*XAPIAN_MOVE_SEMANTICS*/ +#include <xapian.h> + +namespace Mu { + +// LCOV_EXCL_START + +// avoid exception-handling boilerplate. +template <typename Func> void +xapian_try(Func&& func) noexcept +try { + func(); +} catch (const Xapian::Error& xerr) { + mu_critical("{}: xapian error '{}'", __func__, xerr.get_msg()); +} catch (const std::runtime_error& re) { + mu_critical("{}: runtime error: {}", __func__, re.what()); +} catch (const std::exception& e) { + mu_critical("{}: caught std::exception: {}", __func__, e.what()); +} catch (...) { + mu_critical("{}: caught exception", __func__); +} + +template <typename Func, typename Default = std::invoke_result<Func>> auto +xapian_try(Func&& func, Default&& def) noexcept -> std::decay_t<decltype(func())> +try { + return func(); +} catch (const Xapian::DocNotFoundError& xerr) { + return static_cast<Default>(def); +} catch (const Xapian::Error& xerr) { + mu_warning("{}: xapian error '{}'", __func__, xerr.get_msg()); + return static_cast<Default>(def); +} catch (const std::runtime_error& re) { + mu_critical("{}: runtime error: {}", __func__, re.what()); + return static_cast<Default>(def); +} catch (const std::exception& e) { + mu_critical("{}: caught std::exception: {}", __func__, e.what()); + return static_cast<Default>(def); +} catch (...) { + mu_critical("{}: caught exception", __func__); + return static_cast<Default>(def); +} + +template <typename Func> auto +xapian_try_result(Func&& func) noexcept -> std::decay_t<decltype(func())> +try { + return func(); +} catch (const Xapian::DatabaseNotFoundError& nferr) { + return Err(Error{Error::Code::Xapian, "failed to open database"}. + add_hint("Try (re)creating using `mu init'")); +} catch (const Xapian::DatabaseLockError& dlerr) { + return Err(Error{Error::Code::StoreLock, "database locked"}. + add_hint("Perhaps mu is already running?")); +} catch (const Xapian::DatabaseCorruptError& dcerr) { + return Err(Error{Error::Code::Xapian, "failed to read database"}. + add_hint("Try (re)creating using `mu init'")); +} catch (const Xapian::DocNotFoundError& dnferr) { + return Err(Error{Error::Code::Xapian, "message not found in database"}. + add_hint("Try reopening the database")); +} catch (const Xapian::Error& xerr) { + return Err(Error::Code::Xapian, "{}", xerr.get_msg()); +} catch (const std::runtime_error& re) { + return Err(Error::Code::Internal, "runtime error: {}", re.what()); +} catch (const std::exception& e) { + return Err(Error::Code::Internal, "caught std::exception: {}", e.what()); +} catch (...) { + return Err(Error::Code::Internal, "caught exception"); +} + +// LCOV_EXCL_STOP + +/// abstract base +struct MetadataIface { + virtual ~MetadataIface(){} + virtual void set_metadata(const std::string& name, const std::string& val) = 0; + virtual std::string metadata(const std::string& name) const = 0; + virtual bool read_only() const = 0; + + using each_func = std::function<void(const std::string&, const std::string&)>; + virtual void for_each(each_func&& func) const =0; + + /* + * These are special: handled on the Xapian db level + * rather than Config + */ + static inline constexpr std::string_view created_key = "created"; + static inline constexpr std::string_view last_change_key = "last-change"; +}; + + +/// In-memory db +struct MemDb: public MetadataIface { + /** + * Create a new memdb + * + * @param readonly read-only? (for testing) + */ + MemDb(bool readonly=false):read_only_{readonly} {} + + /** + * Set some metadata + * + * @param name key name + * @param val value + */ + void set_metadata(const std::string& name, const std::string& val) override { + map_.erase(name); + map_[name] = val; + } + + /** + * Get metadata for given key, empty if not found + * + * @param name key name + * + * @return string + */ + std::string metadata(const std::string& name) const override { + if (auto&& it = map_.find(name); it != map_.end()) + return it->second; + else + return {}; + } + + /** + * Is this db read-only? + * + * @return true or false + */ + bool read_only() const override { return read_only_; } + + + /** + * Invoke function for each key/value pair. Do not call + * @this from each_func(). + * + * @param func a function + */ + void for_each(MetadataIface::each_func&& func) const override { + for (const auto& [key, value] : map_) + func(key, value); + } + +private: + std::unordered_map<std::string, std::string> map_; + const bool read_only_; +}; + +/** + * Fairly thin wrapper around Xapian::Database and Xapian::WritableDatabase + * with just the things we need + locking + exception handling + */ +class XapianDb: public MetadataIface { +#define DB_LOCKED std::unique_lock lock__{lock_}; +public: + /** + * Type of database to create. + * + */ + enum struct Flavor { + ReadOnly, /**< Read-only database */ + Open, /**< Open existing read-write */ + CreateOverwrite, /**< Create new or overwrite existing */ + }; + + /** + * XapianDb CTOR. This may throw some Xapian exception. + * + * @param db_path path to the database + * @param flavor kind of database + */ + XapianDb(const std::string& db_path, Flavor flavor); + + /** + * DTOR + */ + ~XapianDb() { + if (tx_level_ > 0) + mu_warning("inconsistent transaction level ({})", tx_level_); + if (tx_level_ > 0) { + mu_debug("closing db after committing {} change(s)", changes_); + xapian_try([this]{ DB_LOCKED; wdb().commit_transaction(); }); + } else + mu_debug("closing db"); + } + + /** + * Is the database read-only? + * + * @return true or false + */ + bool read_only() const override; + + /** + * Path to the database; empty for in-memory databases + * + * @return path to database + */ + const std::string& path() const; + + /** + * Get a description of the Xapian database + * + * @return description + */ + const std::string description() const { + return db().get_description(); + } + + /** + * Get the number of documents (messages) in the database + * + * @return number + */ + size_t size() const noexcept { + return xapian_try([this]{ + DB_LOCKED; return db().get_doccount(); }, 0); + } + + /** + * Is the the base empty? + * + * @return true or false + */ + size_t empty() const noexcept { return size() == 0; } + + /** + * Get a database enquire object for queries. + * + * @return an enquire object + */ + Xapian::Enquire enquire() const { + DB_LOCKED; return Xapian::Enquire(db()); + } + + /** + * Get a document from the database if there is one + * + * @param id id of the document + * + * @return the document or an error + */ + Result<Xapian::Document> document(Xapian::docid id) const { + return xapian_try_result([&]{ + DB_LOCKED; return Ok(db().get_document(id)); }); + } + + /** + * Get metadata for the given key + * + * @param key key (non-empty) + * + * @return the value or empty + */ + std::string metadata(const std::string& key) const override { + return xapian_try([&]{ + DB_LOCKED; return db().get_metadata(key);}, ""); + } + + /** + * Set metadata for the given key + * + * @param key key (non-empty) + * @param val new value for key + */ + void set_metadata(const std::string& key, const std::string& val) override { + xapian_try([&] { DB_LOCKED; wdb().set_metadata(key, val); + maybe_commit(); }); + } + + /** + * Invoke function for each key/value pair. This is called with the lock + * held, so do not call functions on @this is each_func(). + * + * @param each_func a function + */ + //using each_func = MetadataIface::each_func; + void for_each(MetadataIface::each_func&& func) const override { + xapian_try([&]{ + DB_LOCKED; + for (auto&& it = db().metadata_keys_begin(); + it != db().metadata_keys_end(); ++it) + func(*it, db().get_metadata(*it)); + }); + } + + /** + * Does the given term exist in the database? + * + * @param term some term + * + * @return true or false + */ + bool term_exists(const std::string& term) const { + return xapian_try([&]{ + DB_LOCKED; return db().term_exists(term);}, false); + } + + /** + * Add a new document to the database + * + * @param doc a document (message) + * + * @return new docid or 0 + */ + Result<Xapian::docid> add_document(const Xapian::Document& doc) { + return xapian_try_result([&]{ + DB_LOCKED; + auto&& id{wdb().add_document(doc)}; + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(std::move(id)); + }); + } + + /** + * Replace document in database + * + * @param term unique term + * @param id docid + * @param doc replacement document + * + * @return new docid or an error + */ + Result<Xapian::docid> + replace_document(const std::string& term, const Xapian::Document& doc) { + return xapian_try_result([&]{ + DB_LOCKED; + auto&& id{wdb().replace_document(term, doc)}; + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(std::move(id)); + }); + } + Result<Xapian::docid> + replace_document(Xapian::docid id, const Xapian::Document& doc) { + return xapian_try_result([&]{ + DB_LOCKED; + wdb().replace_document(id, doc); + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(std::move(id)); + }); + } + + /** + * Delete document(s) for the given term or id + * + * @param term a term + * + * @return Ok or Error + */ + Result<void> delete_document(const std::string& term) { + return xapian_try_result([&]{ + DB_LOCKED; + wdb().delete_document(term); + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(); + }); + } + Result<void> delete_document(Xapian::docid id) { + return xapian_try_result([&]{ + DB_LOCKED; + wdb().delete_document(id); + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(); + }); + } + + template<typename Func> + size_t all_terms(const std::string& prefix, Func&& func) const { + DB_LOCKED; + size_t n{}; + for (auto it = db().allterms_begin(prefix); it != db().allterms_end(prefix); ++it) { + if (!func(*it)) + break; + ++n; + } + return n; + } + + /* + * If the "transaction ref count" > 0 (with inc_transactions());, we run + * in "transaction mode". That means that the subsequent Xapian mutation + * are part of a transactions, which is flushed when the number of + * changes reaches the batch size, _or_ the transaction ref count is + * decreased to 0 (dec_transactions()). * + */ + + /** + * Increase the transaction level; needs to be balance by dec_transactions() + */ + void inc_transaction_level() { + xapian_try([this]{ + DB_LOCKED; + if (tx_level_ == 0) {// need to start the Xapian transaction? + mu_debug("begin transaction"); + wdb().begin_transaction(); + } + ++tx_level_; + mu_debug("ind'd tx level to {}", tx_level_); + }); + } + + /** + * Decrease the transaction level (to balance inc_transactions()) + * + * If the level reach 0, perform a Xapian commit. + */ + void dec_transaction_level() { + xapian_try([this]{ + DB_LOCKED; + if (tx_level_ == 0) { + mu_critical("cannot dec transaction-level)"); + throw std::runtime_error("cannot dec transactions"); + } + + --tx_level_; + if (tx_level_ == 0) {// need to commit the Xapian transaction? + mu_debug("committing {} changes", changes_); + changes_ = 0; + wdb().commit_transaction(); + } + + mu_debug("dec'd tx level to {}", tx_level_); + }); + } + + /** + * Are we inside a transaction? + * + * @return true or false + */ + bool in_transaction() const { DB_LOCKED; return tx_level_ > 0; } + + + /** + * RAII Transaction object + * + */ + struct Transaction { + Transaction(XapianDb& db): db_{db} { + db_.inc_transaction_level(); + } + ~Transaction() { + db_.dec_transaction_level(); + } + private: + XapianDb& db_; + }; + + + /** + * Manually request the Xapian DB to be committed to disk. This won't + * do anything while in a transaction. + */ + void commit() { + xapian_try([this]{ + DB_LOCKED; + if (tx_level_ == 0) { + mu_info("committing xapian-db @ {}", path_); + wdb().commit(); + } else + mu_debug("not committing while in transaction"); + }); + } + + using DbType = std::variant<Xapian::Database, Xapian::WritableDatabase>; +private: + + /** + * To be called after all changes, with DB_LOCKED held. + */ + void maybe_commit() { + // in transaction-mode and enough changes, commit them + // and start a new transaction + if (tx_level_ > 0 && ++changes_ >= batch_size_) { + mu_debug("batch full ({}/{}); committing change", changes_, batch_size_); + wdb().commit_transaction(); + wdb().commit(); + --tx_level_; + changes_ = 0; + wdb().begin_transaction(); + ++tx_level_; + } + } + + void set_timestamp(const std::string_view key); + + /** + * Get a reference to the underlying database + * + * @return db database reference + */ + const Xapian::Database& db() const; + /** + * Get a reference to the underlying writable database. It is + * an error to call this on a read-only database. + * + * @return db writable database reference + */ + Xapian::WritableDatabase& wdb(); + + mutable std::mutex lock_; + std::string path_; + DbType db_; + size_t tx_level_{}; + const size_t batch_size_; + size_t changes_{}; +}; + +constexpr std::string_view +format_as(XapianDb::Flavor flavor) +{ + switch(flavor) { + case XapianDb::Flavor::CreateOverwrite: + return "create-overwrite"; + case XapianDb::Flavor::Open: + return "open"; + case XapianDb::Flavor::ReadOnly: + return "read-only"; + default: + return "??"; + } +} + +static inline std::string +format_as(const XapianDb& db) +{ + return mu_format("{} @ {}", db.description(), db.path()); +} + +} // namespace Mu + +#endif /* MU_XAPIAN_DB_HH__ */ diff --git a/lib/tests/bench-indexer.cc b/lib/tests/bench-indexer.cc new file mode 100644 index 0000000..e76f9f8 --- /dev/null +++ b/lib/tests/bench-indexer.cc @@ -0,0 +1,553 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <glib.h> +#include <string> +#include <thread> +#include <vector> +#include <iostream> +#include <fstream> + +#include <utils/mu-utils.hh> +#include <utils/mu-regex.hh> +#include <mu-store.hh> +#include "mu-maildir.hh" + +#include "utils/mu-test-utils.hh" + +using namespace Mu; + +constexpr auto test_msg = +R"(Return-Path: <htcondor-users-bounces@cs.wisc.edu> +Received: from pop3.web.de [212.227.17.177] + by localhost with POP3 (fetchmail-6.4.6) + for <arne@localhost> (single-drop); Fri, 26 Jun 2020 12:56:08 +0200 (CEST) +Received: from jeeves.cs.wisc.edu ([128.105.6.16]) by mx-ha.web.de (mxweb112 + [212.227.17.8]) with ESMTPS (Nemesis) id 1MdMYE-1jFXaM2gnA-00ZKvt for + <@ID@@web.de>; Fri, 26 Jun 2020 01:28:11 +0200 +Received: from jeeves.cs.wisc.edu (localhost [127.0.0.1]) + by jeeves.cs.wisc.edu (8.14.4/8.14.4) with ESMTP id 05PNLgek013419; + Thu, 25 Jun 2020 18:22:23 -0500 +Received: from shale.cs.wisc.edu (shale.cs.wisc.edu [128.105.6.25]) + by jeeves.cs.wisc.edu (8.14.4/8.14.4) with ESMTP id 05PNLaf0013414 + (version=TLSv1/SSLv3 cipher=AES256-GCM-SHA384 bits=256 verify=OK) + for <htcondor-users@jeeves.cs.wisc.edu>; Thu, 25 Jun 2020 18:21:36 -0500 +Received: from smtp7.wiscmail.wisc.edu (wmmta4.doit.wisc.edu [144.92.197.245]) + by shale.cs.wisc.edu (8.14.4/8.14.4) with ESMTP id 05PNLaMK013694 + (version=TLSv1/SSLv3 cipher=DHE-RSA-AES128-GCM-SHA256 bits=128 + verify=NO) + for <htcondor-users@cs.wisc.edu>; Thu, 25 Jun 2020 18:21:36 -0500 +Received: from USG02-CY1-obe.outbound.protection.office365.us + ([23.103.209.108]) by smtp7.wiscmail.wisc.edu + (Oracle Communications Messaging Server 8.0.2.4.20190812 64bit (built + Aug 12 + 2019)) with ESMTPS id <0QCI042LC8VUXFC0@smtp7.wiscmail.wisc.edu> for + htcondor-users@cs.wisc.edu (ORCPT htcondor-users@cs.wisc.edu); Thu, + 25 Jun 2020 18:21:31 -0500 (CDT) +X-Spam-Report: IsSpam=no, Probability=11%, Hits= RETURN_RECEIPT 0.5, + FROM_US_TLD 0.1, HTML_00_01 0.05, HTML_00_10 0.05, SUPERLONG_LINE 0.05, + BODYTEXTP_SIZE_3000_LESS 0, BODY_SIZE_10000_PLUS 0, DKIM_SIGNATURE 0, + KNOWN_MTA_TFX 0, NO_URI_HTTPS 0, SPF_PASS 0, SXL_IP_TFX_WM 0, + WEBMAIL_SOURCE 0, WEBMAIL_XOIP 0, WEBMAIL_X_IP_HDR 0, __ANY_URI 0, + __ARCAUTH_DKIM_PASSED 0, __ARCAUTH_DMARC_PASSED 0, __ARCAUTH_PASSED 0, + __ATTACHMENT_SIZE_0_10K 0, __ATTACHMENT_SIZE_10_25K 0, + __BODY_NO_MAILTO 0, + __CT 0, __CTYPE_HAS_BOUNDARY 0, __CTYPE_MULTIPART 0, __HAS_ATTACHMENT 0, + __HAS_ATTACHMENT1 0, __HAS_ATTACHMENT2 0, __HAS_FROM 0, __HAS_MSGID 0, + __HAS_XOIP 0, __HIGHBITS 0, __MIME_TEXT_P 0, __MIME_TEXT_P1 0, + __MIME_TEXT_P2 0, __MIME_VERSION 0, __MULTIPLE_RCPTS_TO_X2 0, + __NO_HTML_TAG_RAW 0, __RETURN_RECEIPT_TO 0, __SANE_MSGID 0, + __TO_MALFORMED_2 0, __TO_NAME 0, __TO_NAME_DIFF_FROM_ACC 0, + __TO_NO_NAME 0, + __TO_REAL_NAMES 0, __URI_IN_BODY 0, __URI_MAILTO 0, __URI_NOT_IMG 0, + __URI_NO_PATH 0, __URI_NS , __URI_WITHOUT_PATH 0 +X-Wisc-Doma: @ID@X@numerica.us,numerica.us +X-Wisc-Env-From-B64: d2VzbGV5LnRheWxvckBudW1lcmljYS51cw== +X-Spam-PmxInfo: Server=avs-13, Version=6.4.7.2805085, + Antispam-Engine: 2.7.2.2107409, Antispam-Data: 2020.6.25.231519, + AntiVirus-Engine: 5.74.0, AntiVirus-Data: 2020.6.25.5740002, + SenderIP=[23.103.209.108] +X-Wisc-DKIM-Verify: @ID@XXXXXXX@numerica.us,numericaus.onmicrosoft.com!pass +X-Spam-Score: * +ARC-Seal: i=1; a=rsa-sha256; s=arcselector5401; d=microsoft.com; cv=none; + b=KyXoddJsnsHsBwhdlO5rcljgMRaylJAUAxWTjG4jQL1C8XJAMgeERtH2sRffdjibYUFfSuDUNJmrTrvrbjKGUt2I8J2M2MgUB/upMoroVPNBrP1Fy9wMeZJQuSS4r4KjZZktsl2i8eq667pzOZO6+wX2IA5M7YtxDqglcWOE6btWzbABVjx+9eCXMt0eMd1+UI6ABK8Frd33EFQLKT0h/cxidWR9l+0gCMAcRxsLrQ82+ckU606AIV/DA1E4Tq7ADe/+CRv4QszDN93pWL/1N2/OOh9vFTs9g9ZG6uXjN+Km/IAdylPbfHgKW60ev3/Bvv6N3pA7DjpuiKj6BnW7mQ== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; + s=arcselector5401; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; + bh=OZrj1we1ZUH0xBMhJ5/F6EQnB0cmitFs2xZW1fLMRNs=; + b=Pq07a3u26s2UdpucJuVQ0h68272wx46Wp61x/30TelPPFLCRxVjmlH1U3IBmIsZ1jOEtGXFJRv65L3HmwGxRUdLlMOdPRB64BBfHQ9NGWUBykKQmOrJNGJs635nEdpugpzngzIdcg1PS5vHxPJAnOeqoo71OVPI3JqPrPEn2TJJgb9J6PApexkqIbVl35prGPsyS/t2IlYw3/ihWzORG6wvqJeqedgpJTBXeGaDoMa+MQ1BeUsdvybh8+hau4ASpM5lwyeXlGmJ5mUTZi39jp+dFdDrmCj/VM4ezeuXeH9+HFtDjKLZJaTDWUID0IBcr91BaoQE/4r6y+lpkah6LLQ== +ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=pass + smtp.mailfrom=numerica.us; + dmarc=pass action=none header.from=numerica.us; + dkim=pass header.d=numerica.us; arc=none +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=numericaus.onmicrosoft.com; s=selector1-numericaus-onmicrosoft-com; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; + bh=OZrj1we1ZUH0xBMhJ5/F6EQnB0cmitFs2xZW1fLMRNs=; + b=cFn0eL5k2IKry9U8qa8mbVaxRiyicUAWzRc3NUtj+VEbgShfrz8SO6FPX20WTQQJg/Fu/3isqsSEUt+9NSEEbgd5eQ1EVz5E/JVeNjPe9GXR0JEF/g3f6yM7CO+kKTvXSRvQjce683U0j7Aj1pSDEktoVNP4xvOS2Gx9VjdWTmc= +Received: from DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM (2001:489a:200:413::10) + by DM3P110MB0490.NAMP110.PROD.OUTLOOK.COM (2001:489a:200:413::14) + with Microsoft SMTP Server + (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) + id 15.20.3109.25; Thu, 25 Jun 2020 23:21:07 +0000 +Received: from DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM + ([fe80::f548:f084:9867:9375]) by DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM + ([fe80::f548:f084:9867:9375%11]) with mapi id 15.20.3131.024; Thu, + 25 Jun 2020 23:21:07 +0000 +From: Raul Endymion <XXXXXXXXXXXXXXXXXXXX@numerica.us> +To: "'htcondor-users@cs.wisc.edu'" <htcondor-users@cs.wisc.edu> +Thread-topic: OPINIONS WANTED: Are there any blatent downsides I am missing to + the following Condor configuration +Thread-index: AdZLRbEvYoEDBZChS62aOHgPzKD8kw== +Date: Thu, 25 Jun 2020 23:21:06 +0000 +Message-id: <DM3P110MB04746CDBA55B3E597EFD1877FA920@DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM> +Accept-Language: en-US +Content-language: en-US +X-MS-Has-Attach: yes +X-MS-TNEF-Correlator: +X-Originating-IP: [50.233.29.54] +x-ms-publictraffictype: Email +x-ms-office365-filtering-correlation-id: f4edecdf-5582-4b2d-226a-08d8195e7007 +x-ms-traffictypediagnostic: DM3P110MB0490: +x-microsoft-antispam-prvs: <DM3P110MB04907C313C4243B4FE42A20CFA920@DM3P110MB0490.NAMP110.PROD.OUTLOOK.COM> +x-ms-oob-tlc-oobclassifiers: OLM:10000; +x-forefront-prvs: 0445A82F82 +x-ms-exchange-senderadcheck: 1 +x-microsoft-antispam: BCL:0; +X-Forefront-Antispam-Report: CIP:255.255.255.255; CTRY:; LANG:en; SCL:1; SRV:; + IPV:NLI; SFV:NSPM; H:DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM; + PTR:; CAT:NONE; SFTY:; + SFS:(346002)(366004)(6916009)(83380400001)(71200400001)(8936002)(55016002)(5660300002)(8676002)(9686003)(66616009)(64756008)(33656002)(52536014)(66446008)(66476007)(66556008)(66946007)(99936003)(76116006)(186003)(86362001)(26005)(7696005)(6506007)(44832011)(508600001)(2906002)(80162005)(80862006)(491001)(554374003); + DIR:OUT; SFP:1102; +x-ms-exchange-transport-forked: True +MIME-version: 1.0 +X-OriginatorOrg: numerica.us +X-MS-Exchange-CrossTenant-Network-Message-Id: f4edecdf-5582-4b2d-226a-08d8195e7007 +X-MS-Exchange-CrossTenant-OriginalArrivalTime: 25 Jun 2020 23:21:06.8341 (UTC) +X-MS-Exchange-CrossTenant-fromentityheader: Hosted +X-MS-Exchange-CrossTenant-id: fae7a2ae-df1d-444e-91be-babb0900b9c2 +X-MS-Exchange-CrossTenant-mailboxtype: HOSTED +X-MS-Exchange-Transport-CrossTenantHeadersStamped: DM3P110MB0490 +Subject: [HTCondor-users] OPINIONS WANTED: Are there any blatent downsides I + am missing to the following Condor configuration +X-BeenThere: htcondor-users@cs.wisc.edu +X-Mailman-Version: 2.1.19 +Precedence: list +List-Id: HTCondor-Users Mail List <htcondor-users.cs.wisc.edu> +List-Unsubscribe: <https://lists.cs.wisc.edu/mailman/options/htcondor-users>, + <mailto:htcondor-users-request@cs.wisc.edu?subject=unsubscribe> +List-Archive: <https://www-auth.cs.wisc.edu/lists/htcondor-users/> +List-Post: <mailto:htcondor-users@cs.wisc.edu> +List-Help: <mailto:htcondor-users-request@cs.wisc.edu?subject=help> +List-Subscribe: <https://lists.cs.wisc.edu/mailman/listinfo/htcondor-users>, + <mailto:htcondor-users-request@cs.wisc.edu?subject=subscribe> +Reply-To: HTCondor-Users Mail List <htcondor-users@cs.wisc.edu> +Content-Type: multipart/mixed; boundary="===============0678627779074767862==" +Errors-To: htcondor-users-bounces@cs.wisc.edu +Sender: "HTCondor-users" <htcondor-users-bounces@cs.wisc.edu> +Envelope-To: <@ID@XXXXX@web.de> +X-UI-Filterresults: unknown:2;V03:K0:cdojl5YHfkg=:jhTbQXp38SL2za/LB4M7aUwpyw + 5rDHoN1+/ScH/O9/G1fKWbGryQ203thF+1ZrHUOOwq8MVOc5SsoqzSTsaNbEAdthFcDDz3Oui + SHxX1hdpV3UOjZEHzWlpjEjRe7t74g2RI/ESELmkPuLg/LZC7SjAsg70cTJBIfDPYxJkJAcUl + 9W6OEBsmtTDO0va/EQRYjfkpoF9tjfmfMNw9KSKHuDdqZu2Xfak8mQKnWsoxWeUkD31r60iPC + yikbj7KP5AlHaWMzyTTdlvtjYRLfSuUSe1uqjI5NWCnZDDjz7zODoaWPp7p2U/MQenXEjN6+M + WnZL6ZC8AGtze/hYgOCXcLf4ydQ7m9YueJiY5nDn7g+cwnhxypVNFTL5NjSpKKXbkzbyu9Tdl + ez+92g/9pGW17iOo5NrFtfctLlmCEH0RxjouKI7FBmv3bIvFC4FvfghiNf7OZmRg2/nT5i+1o + AICYNAx2y5CezKsKM2f1tm60dkydQIR8pK45dDKZPz3i7NeJm9dknZ2OYFTnucUvdPaT8nR43 + cK3kk2QUE48Ngo/0NwepSGrV9TkOt+hY3PUYkXWp/mwP2QPSjy4cALyvLyKwG24qZ9CiiRLMV + KqPFlCRnoDG5MHJ4d0krFlqmg8rNsWzV3oWfMNKFZmD24lVUmWGb+ExxbCFc0xzIt12o/EqBw + nVkXLu+E0apM+cmG6ubYfOymRoUpiKsZI9ivc+mAaEE+v2RBzcAURzlhzQHIn81onvzbQwCge + tMtBkSEfqyoa1HjalX4B9WQ90M42K+7xW039ydakQ7JOeYVpkPXYoBF7mbrXRckhMXjQatLQ8 + MWA8+U031Xfa1ueOIfCCkzJ49wyx1LoLPyqdjCvnzaRd72yNEMJ5zM/itMIPE9reIHBtpom0i + RhIYdJFDrL+SKqE68lcJakCcF3R+VLApwLKOr0HChGQjdEk7c/rm5E0dF5f3oYlHf591QoXIJ + h16yfcJYe6fMo1YYunkvbEDFPpzttIq7aIk0FzxrOdRvj3yQajDbwOpYI/5T/DaabPn3M8lK7 + 8pn7LrbmyCaLHhkYMS4h3SDkYWsifza6vkldizrK7IPf6KhS7AhTkbnEonWS6454GLUg1nYGX + W5Qp/G9LzvjtEGQMcwnCN5jb5zq7o3f+9FrROKjpwFxL+mL85CEXY/KMOVpf6hDJJfSyu6/X6 + FpbwlJVLFdGeA0/+xcKcmutpkJACgK2kHqvZ8MZxt+5jBJWVlIDLZKa8/IoGWC+ikLX2/hPNB + 4TU89QYG5ygPmwwDXruFG7N7jVURZceHqWNKtqegS6YQ5nirsPJWJR7jzgr+HbntUaQETXNpn + QrxpsVHXfqRu2GlP5h28RaIpvBVUcwqrs+eLJELStvBzyAmaVPVoKFjEWFfwrmE89W6Bmz2W3 + kHExOq3hI3gDsGXKjTjT/kjHkaHmtnVUXr4vqovf8Ht4Vwmtf0S4xsgpYjnYjUIzG9eiwIFAZ + hL2gvjwW51qtMvybf01C50xTiS9GSfO0SR7meBPA67skcA+wFo11wmwXsUk1irpKnC+Y9hVZX + 2vPkfZ1T2VXNo997cQC59lBpi/TU5gnuM7H/Vcl7tF3Lqtmqut7s6HkPWCegDZ3O2W7shH7aZ + 1bOXbO+W/SNC+WcMnj+fhuP+dHcrt0Vw4RD9knJOOzdZTH3OCli/vpjqgTbCKEaWMhCIeM2g0 + RiLFxTeTEEBCa49bwa8n2r4T/vA3duZd8F/DNKvWTfhRr1Mxtz3n15EOar13fFijtnieEiv4/ + vO/5uRF+H86Fcoua7B8AswThbiG1vou6M48g0Zo6iGEcrueKEaHMI4XM7wQF77KazMdn5f1BP + +KyQX83aHJN/qGniXgF8yu+h0M7Nf0YrTteYQd2C/HZrIA8IaLqqvLoGRl7dRBnbZiP7jRdQm + 1YEYtjX4XBoShrXPfIxPnJBUBnnOaePYxOJkS2FaBv19jPkMnyc9xuJYD7JOTFnXKzAnoaBqT + OR+dGrLLGZ1MM/0gqclKTv7Hcce+6CJyTWkx5mq42w49HFI/kdHBRxU8xIRv4B9l0ePf9EbWr + cDcrssee//6KXiRmF4fm7jq828/uhj8MIJet9sIU5ncKwHEse3I4YmVT5+dB+ZGZh0gbJPFj6 + xcICpshhYct+euMCdNfy3lkxiRr76RwfBzLAOP5+1U3GAx/hcsL2AgyBHMwWo+Kkeq8pPy4YI + pQMxJyylI6JMa/DbBggnDk+xNZpRKo/XA4lAJY57DCOPL2ZcL8kU2aCd5LjtYHK0ZWSFtOjxs + oIEr/f2vvg+zibxzaANBzylZn3yPe9pI/IBefu9fL4MVaYY3aboxuncX4fyi0VH0WbFkSYXRi + a7LIu3LI2LTU13C/LE7j9hmxP6TApyiXi14f0GSa2sbF6HWp2v2rhYM7h67AAn3SQgvcJLpgb + Hz5ABb/OAk6ABVEl+a483zexJ6iT2P0gYc08zmewy8Jf8AD9r846k9pGZuhBaOHREx3bA16Bj + uWYh3QzSI6MQoJM3XbBGLVkX36Lfj54T9kk97lLaxfbGPuNoyOV9iTBKxts3m2KD+52iH3EEi + glbH6HNIUHyCHdEXsXyGVFwfM9V7OQcVO/g266KIQ74wU16x/Zdsq4p/1PcRXHRnoMxP/pUrj + EOLWzFU71qzC/OSkYWRil9HXUyucTFGQ0N08jZNXctI9lElWtgq3iI+Cz2F20rz+LJGhSHSkZ + 0G5JgXrtspeJN5yoH6TOE0hblr5sZcAM0wiSP7x/hPBeYHswzTA5/laWMn++9aTPVgpPaJ9/x + wyLm55OZr4Jl+StWd3MqLCgiRB3cNGrDX7f8Eqnj4wfCHiGIUHewD4qrfXraZQhIk17W+9JyD + osmUiVD9ZRdNCY2eNnu8ZkJ4uzKl44lwLL43sInKBjdAHlnoxrR2FOrYXbnU31ujwxdeUr6Hs + xPFy0Git0CpWCWYmaz37KA8GW7PE4ffWzcfCmz6AKBrbHcCreeUnyqnSEDy9ubnz7mcLRnu3W + RAWi6diI8gcS9g0+r4z5PtZX9rveXRekHJ4k08VuYVmdiz3gjXmHPlm9IKPEAbygP2EYgjwGE + RbReLc8xHJlfLbwdXyGw0HU= + +--===============0678627779074767862== +Content-language: en-US +Content-type: multipart/signed; protocol="application/x-pkcs7-signature"; + micalg=2.16.840.1.101.3.4.2.3; + boundary="----=_NextPart_000_0018_01D64B14.F58791A0" + +------=_NextPart_000_0018_01D64B14.F58791A0 +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +Hey! + +I am architecting our final HTCondor configuration over here and I have = +an idea I am unsure about and I would like to ask some experienced users = +for their opinion. + +Background, we have a small, relatively homogenous cluster (with no = +special universes) and less than 10 users. Since each user has their own = +workstation separate from our cluster I thought the following = +configuration would suit our needs, but I want to make sure there isn't = +a huge disadvantage I am missing: + +1. Set the Central Manager to be highly available to the point of = +tolerating N cluster machine failures +2. Put a Submit on each of the users' workstations (I am a little = +worried about the resource usage of the condor_shadow and condor_schedd, = +my users are already running into RAM consumption issues over time as it = +is) +3. Place an Execute on each of the cluster machines, which would lead to = +the central manager being on a machine that is also executing jobs + +Fortunately both my users' and cluster machines all have access to the = +same network storage, and we have centralized authentication so we can = +just use our users' credentials to authenticate everywhere.=20 + +Before I set this in dry mud, does anyone have any retrospective = +recommendations I could benefit hearing from, since I am still pretty = +new to the project? + +Thank you! +-Raul + +Raul Endymion =E2=80=93 Cluster Manager +Numerica Corporation (www.numerica.us) +5042 Technology Parkway #100 +Fort Collins, Colorado 80528 +=E2=98=8E=EF=B8=8F (970) 207 2233 +=F0=9F=93=A7 @ID@XXXXXXXXXXXXXXX@numerica.us + + + +------=_NextPart_000_0018_01D64B14.F58791A0 +Content-Type: application/pkcs7-signature; + name="smime.p7s" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="smime.p7s" + +MIAGCSqGSIb3DQEHAqCAMIACAQExDzANBglghkgBZQMEAgMFADCABgkqhkiG9w0BBwEAAKCCEv4w +ggWpMIIDkaADAgECAhAV2Tfkh0+gtEu0gskeSMTdMA0GCSqGSIb3DQEBCwUAMFsxEjAQBgoJkiaJ +k/IsZAEZFgJ1czEYMBYGCgmSJomT8ixkARkWCG51bWVyaWNhMRIwEAYKCZImiZPyLGQBGRYCYWQx +FzAVBgNVBAMTDmFkLUdJTEdBTEFELUNBMB4XDTE2MDcyNDE5NTcxM1oXDTM2MDcyNDIwMDcxMlow +WzESMBAGCgmSJomT8ixkARkWAnVzMRgwFgYKCZImiZPyLGQBGRYIbnVtZXJpY2ExEjAQBgoJkiaJ +k/IsZAEZFgJhZDEXMBUGA1UEAxMOYWQtR0lMR0FMQUQtQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQCq+/935KPrc8clxrq76k7GrrUHRbsM4FCfyrWicGPZsOKbJfcoloF2EAfj6AYR +QyU/l9um/8NqW+cu6/TY6YcY622L+UtT1QWC/Kt0kVL7cTtZN+VK/BkjcDVbUOqdeFY1q0tMzdco +WFxqjayGRYnX6oEZ7krDsGtJBBET/504Z3vDq/0ZD3lNG2dCWp1y+3VzUcb+OKkOPwMGHpw3gZM5 +lZN/znB7d7qwxFSRoLzZZB3nZKKJHcp2ZuyJR+pCT5VdHGGV4gpVQKuL49/UoJBA0o8Kv0DGPByD ++LVwhlyFMi2jlnCd5lqiWRw9JAE3fqS/Di/cGbMjXMI2CplBj+GmZH8fgy4BQRwmsOUELTaYkJyJ +otcHGENO1+xYrR/lFEQLhh+8V2IJvBM2G1dgJ3EuEslL4q0xGeYLZJd7Z9xvXkAJaX/eWjHWICFI +zbsH/6fBqXYow/V8hfZhb20dGGnPESXPqMv/1mLgUIqr++Fjl6zKM5mYZuHlmrtd+eLgg7VsjDvh +cMxdQnju+jzJflxlmY2KSwt5lsu7viqmQyqVUnHFaEsV116B0uCROc5o1pBdRMdeeLrRoj6xPVlc +IzmIZz3wZERxCAWeJqBx5d1kXe+cDL4pMNQ/hmah4mshjtyOGv+oEgcdxzUQ72W7JNLhSv8C6gpU +eQwPq8usFAvUOwIDAQABo2kwZzATBgkrBgEEAYI3FAIEBh4EAEMAQTAOBgNVHQ8BAf8EBAMCAYYw +DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUF+CLMX/eZk96ElRSeiEHqnsujqEwEAYJKwYBBAGC +NxUBBAMCAQAwDQYJKoZIhvcNAQELBQADggIBACcwALtn+SFUx+YTrLCFY+Ghh4yubQt3YdEI6hOQ +JnmNPKsUEzCvoRE5L2ZLkG2VhJNX3KAJmXgkZMCGBPbiA/65r/cbYqZATQEG/g9aVicz/IBHXvg4 +7+YDDN9VpRy8c93AZNNTRf83Pw+CDsdIGG7mg8rc0tiCgt0V3gN0wF8oRSsb/trqd+ujk41bvaPw +Rl+8JUeRN0Pq9lH4VGGk9GEIQv8JXhr2VKFmJcGKLB+qvMRvWQZ5oPTGDE3pUYI5q8f7/fMiJKU6 +hb9l+tXP7uDLWIawg/MoUc2BwAThyXFk9LZhkYWYpzbaf2Ez2JYieD4ey8RjEKvis9mF6Z/p6+69 +GbYvuf2bRikYenrmboXCUO820totjP2UyHczexZsMP/XznmyDJuN+BDLzLjm7ks8lXDwpF/Kqnjm +1EyiQI0OB4cn889yM039U7raJeHpuiwju2/YO6krE+plLQhkM7pl6v6Ly/ZKICwDfbcU8k8LE4+K +3VaXmVYRYbSXx8l2Ke0CWKNfehBGQ024gKjNt8t7gCgInG5s+roumqeKyfCWlhYll1FAxEQmwP/6 +966y7uJrGLra0VUjdppbZpAENSF0pdX08VfsasSZ20hnCaLWO1b3i0ZOBLBAoNzeCm+BdS6DAOhy +JnHHZ+OBoiaYwCSjSvTDmHyQkNK3wmu+/wyNMIIGnDCCBISgAwIBAgITbwAAAEFhCq43is5OqAAA +AAAAQTANBgkqhkiG9w0BAQsFADBbMRIwEAYKCZImiZPyLGQBGRYCdXMxGDAWBgoJkiaJk/IsZAEZ +FghudW1lcmljYTESMBAGCgmSJomT8ixkARkWAmFkMRcwFQYDVQQDEw5hZC1HSUxHQUxBRC1DQTAe +Fw0xOTA3MjIxNDE4MDFaFw0yMTA3MjIxNDI4MDFaMFwxEjAQBgoJkiaJk/IsZAEZFgJ1czEYMBYG +CgmSJomT8ixkARkWCG51bWVyaWNhMRIwEAYKCZImiZPyLGQBGRYCYWQxGDAWBgNVBAMTD2FkLUNF +TEVCUklBTi1DQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRLgjg0yC0P2jLwTCIA +V/zEGk/PEc3pZxNAo7m0I/SXdNulUEkjxai5Wq53i0EhWVLpUU8XY3joXax46yCMqh0PUn90QmMD +BybLyFDX6av8tVS5cQs0HbTZdIuj7A/dsKzKKIrSHd3SQ9MLNPRkSRdhagmf5LCF1Y4xEEiuAA/H +XdYAxGIcl8n6b2CcLlZzq4W13Ipv8FIZoDsG1u0b9NGfeSOOHidi5kdD6r8lM5PaSPmZsl5PdKK6 ++E1Y6rBCvITu0MBo5Tjuwt5cok3Ve0BK5Fg89aIL2/rMicm20qG6nbqxLhHeR0mhPO98KIIzDoeL +rLpAlWS7GoPvJqbRzxsCAwEAAaOCAlYwggJSMBAGCSsGAQQBgjcVAQQDAgEBMCMGCSsGAQQBgjcV +AgQWBBSv5TU1Bjnw5n3u1iO2y+BHQXk7MTAdBgNVHQ4EFgQUoeMyqBhiyBcgwJN8zbr7pRbgs+sw +GQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMB +Af8wHwYDVR0jBBgwFoAUF+CLMX/eZk96ElRSeiEHqnsujqEwgdMGA1UdHwSByzCByDCBxaCBwqCB +v4aBvGxkYXA6Ly8vQ049YWQtR0lMR0FMQUQtQ0EsQ049R2lsZ2FsYWQsQ049Q0RQLENOPVB1Ymxp +YyUyMEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24sREM9YWQsREM9 +bnVtZXJpY2EsREM9dXM/Y2VydGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNlP29iamVjdENsYXNz +PWNSTERpc3RyaWJ1dGlvblBvaW50MIHGBggrBgEFBQcBAQSBuTCBtjCBswYIKwYBBQUHMAKGgaZs +ZGFwOi8vL0NOPWFkLUdJTEdBTEFELUNBLENOPUFJQSxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNl +cyxDTj1TZXJ2aWNlcyxDTj1Db25maWd1cmF0aW9uLERDPWFkLERDPW51bWVyaWNhLERDPXVzP2NB +Q2VydGlmaWNhdGU/YmFzZT9vYmplY3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MA0GCSqG +SIb3DQEBCwUAA4ICAQBmRoSlPe++k7tsAJOvq0+0dNI6yk6gOBmY4g5jL9NTEjSxPWkeYegIwLr2 +UqpiIIZmAh9e9v3z0T2egVyRqNezLPXLkg/2gUfV6D0kRyKtG5mL0yAn/0hkkVyf6jWJpCKmH77x +0w3UpnfKs79jv5YpQDhC2eRFivN50HhIkigLWScPq4zd81ghmN8VFTHVQmsGua/mm1Oj5/pBFuQF +B4ljon1N//wX5ZJZaUlJR9eR9tM9m+Gyds2flr5+mZT6Zgm26fKiC5zs91aGnzqGx6s30jfXELP2 +FjFrrR46ooV7ehhnyBlCACxIWqXe5sSZsSh9oEYZ7Ux5Vq0thkfArBWsF7HA+LovKCUyHLcXbVBB +6/VAwZ3GLYi/bqbVIEFlVRu4nv/JyKWwoGbAhGyzZNWoeHszFrEIQbQMoMsEumVkMZreE6AxP+zb +6JPPOjlhpymtMo54z1MDYJPyo4HmcpL4xUjHZgqgOxMrbHC4oIVLvKZ/scbVBhPnd0tHHSZqj3ZS +gfTvG/ut/tLNTXXe48PkLBw4KguhbLm61Elu3wJALT0UL+ENgUWwb7csUGQBqOyPAHXGYnf/ACOc +UBqQckcrK8Jq3u8rnCloW3uDw86hw7MFM+YjmhVRdYRxpJmhKVPT6Amufp2WsSVId8q3CSqTH33L +fcxbV1n7hLWHA67MhTCCBq0wggWVoAMCAQICEycAAAsJMaw2RjtHZFUAAQAACwkwDQYJKoZIhvcN +AQELBQAwXDESMBAGCgmSJomT8ixkARkWAnVzMRgwFgYKCZImiZPyLGQBGRYIbnVtZXJpY2ExEjAQ +BgoJkiaJk/IsZAEZFgJhZDEYMBYGA1UEAxMPYWQtQ0VMRUJSSUFOLUNBMB4XDTIwMDUxMjE1MDk0 +MloXDTIxMDcyMjE0MjgwMVowgcExEjAQBgoJkiaJk/IsZAEZFgJ1czEYMBYGCgmSJomT8ixkARkW +CG51bWVyaWNhMRIwEAYKCZImiZPyLGQBGRYCYWQxETAPBgNVBAsTCE51bWVyaWNhMQ4wDAYDVQQL +EwVVc2VyczEYMBYGA1UECxMPUHJlc2VudCBJbnRlcm5zMRYwFAYDVQQDEw1XZXNsZXkgVGF5bG9y +MSgwJgYJKoZIhvcNAQkBFhl3ZXNsZXkudGF5bG9yQG51bWVyaWNhLnVzMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA5clDLapXkiLVXhAFP9GJv+JJkt+cacyvWaX9xEvqMQXOXb7MqO5E +DJE8XPMfxaX84WhuMMePOc9SNUKpDtTa2SHz+AOom+JH38ce2gfrdOPwez/e6RrUb3o8ZvMr3hJl +Yy+6vEFEADIICfHSlIjkLJbGNFTRDccvkOPjD2W+fmzFAtWyNb/eqM+mwdTuXjOxTvP6V34zJsvc +YKJUzhhD8jI7GdqOoNoirTlaMVTH5udK0P2KvzD6F0LfwcOlc3bTvY9uI585xhdniK4yAIka8OMq +5zmyEQLYOadcVSscjAlkC1sQ0gbwL3AdwS+bntryq+2Ds380OJ+Z1Uy7TRkeBQIDAQABo4IDADCC +AvwwPAYJKwYBBAGCNxUHBC8wLQYlKwYBBAGCNxUI9/Bss4wDhbmBGISeqheH4YBfgSWC6qJEgcjE +IgIBZQIBKDATBgNVHSUEDDAKBggrBgEFBQcDBDAOBgNVHQ8BAf8EBAMCBaAwGwYJKwYBBAGCNxUK +BA4wDDAKBggrBgEFBQcDBDBEBgkqhkiG9w0BCQ8ENzA1MA4GCCqGSIb3DQMCAgIAgDAOBggqhkiG +9w0DBAICAIAwBwYFKw4DAgcwCgYIKoZIhvcNAwcwHQYDVR0OBBYEFDZHoDwoOKD5uzpF/2CcZSeg +XWLmMB8GA1UdIwQYMBaAFKHjMqgYYsgXIMCTfM26+6UW4LPrMIHVBgNVHR8Egc0wgcowgceggcSg +gcGGgb5sZGFwOi8vL0NOPWFkLUNFTEVCUklBTi1DQSxDTj1DZWxlYnJpYW4sQ049Q0RQLENOPVB1 +YmxpYyUyMEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24sREM9YWQs +REM9bnVtZXJpY2EsREM9dXM/Y2VydGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNlP29iamVjdENs +YXNzPWNSTERpc3RyaWJ1dGlvblBvaW50MIHHBggrBgEFBQcBAQSBujCBtzCBtAYIKwYBBQUHMAKG +gadsZGFwOi8vL0NOPWFkLUNFTEVCUklBTi1DQSxDTj1BSUEsQ049UHVibGljJTIwS2V5JTIwU2Vy +dmljZXMsQ049U2VydmljZXMsQ049Q29uZmlndXJhdGlvbixEQz1hZCxEQz1udW1lcmljYSxEQz11 +cz9jQUNlcnRpZmljYXRlP2Jhc2U/b2JqZWN0Q2xhc3M9Y2VydGlmaWNhdGlvbkF1dGhvcml0eTBS +BgNVHREESzBJoCwGCisGAQQBgjcUAgOgHgwcd2VzbGV5LnRheWxvckBhZC5udW1lcmljYS51c4EZ +d2VzbGV5LnRheWxvckBudW1lcmljYS51czANBgkqhkiG9w0BAQsFAAOCAQEAX3zFhiDYU+vQap2J +hiysyC9L7nkL7VI2OQWg4Z/JnNJTFiA6BwtoDYAT4qq1Jix4hZc+g78Gj99OnkhlBQDe9Hq12yI9 +muboQSDAYO6iDK76wQv3Rt8Fl4SUD4Ygwy52QrkTDrj/HZxTNask5p/2ilGBJnG9KT2VbEgGJkP9 +kXn1vAgOl3BCxgjdWekWCvxpmffr+Z3UtmQIiZAB3OsKcgdsSy9pveTMjxtKJemaH3kpXQiTgCev +CMuWZb3YnqXI8Fd+uUw6HwA4c+ZH62G9Q8KGkwXyhOPizmm3UeSlMo27yUCE+cF5EIHBxpGJ6z83 +7MbxMVKnS1Wz1n8MtW2ezDGCBCEwggQdAgEBMHMwXDESMBAGCgmSJomT8ixkARkWAnVzMRgwFgYK +CZImiZPyLGQBGRYIbnVtZXJpY2ExEjAQBgoJkiaJk/IsZAEZFgJhZDEYMBYGA1UEAxMPYWQtQ0VM +RUJSSUFOLUNBAhMnAAALCTGsNkY7R2RVAAEAAAsJMA0GCWCGSAFlAwQCAwUAoIICfzAYBgkqhkiG +9w0BCQMxCwYJKoZIhvcNAQcBMBwGCSqGSIb3DQEJBTEPFw0yMDA2MjUyMzIwNDRaME8GCSqGSIb3 +DQEJBDFCBEBaj66vdgjAhEO0p7lO6X44h+LpUlAcROa5Hi4Jp5aWS4hU8CuqOrH12y2GRNmNhKLa +0YieL4fCL3YqDRfop79NMFIGCyqGSIb3DQEJEAIBMUMwQQQdAAAAABAAAACgLzslsB99TKIYKeHy +Wh5cAQAAAACAAQAwHTAbgRl3ZXNsZXkudGF5bG9yQG51bWVyaWNhLnVzMIGCBgkrBgEEAYI3EAQx +dTBzMFwxEjAQBgoJkiaJk/IsZAEZFgJ1czEYMBYGCgmSJomT8ixkARkWCG51bWVyaWNhMRIwEAYK +CZImiZPyLGQBGRYCYWQxGDAWBgNVBAMTD2FkLUNFTEVCUklBTi1DQQITJwAACwkxrDZGO0dkVQAB +AAALCTCBhAYLKoZIhvcNAQkQAgsxdaBzMFwxEjAQBgoJkiaJk/IsZAEZFgJ1czEYMBYGCgmSJomT +8ixkARkWCG51bWVyaWNhMRIwEAYKCZImiZPyLGQBGRYCYWQxGDAWBgNVBAMTD2FkLUNFTEVCUklB +Ti1DQQITJwAACwkxrDZGO0dkVQABAAALCTCBkwYJKoZIhvcNAQkPMYGFMIGCMAsGCWCGSAFlAwQB +KjALBglghkgBZQMEARYwCgYIKoZIhvcNAwcwCwYJYIZIAWUDBAECMA4GCCqGSIb3DQMCAgIAgDAN +BggqhkiG9w0DAgIBQDALBglghkgBZQMEAgMwCwYJYIZIAWUDBAICMAsGCWCGSAFlAwQCATAHBgUr +DgMCGjANBgkqhkiG9w0BAQEFAASCAQBNFxhcbK6Rmw0Xyu+79cH5kUsXENcdUaJPKlegcY/gl2BZ +0CPpGcRnwz6z8OPYjvw3jrkiAE8nBbuCKu1CPtuk1h4Cybk7exyMybYvK5xge+N+dz2mFipRfGSY +rl/ztX1jyvcDruxaSJwb8WMhAGs505yfaCJfwgFOI3QGi+wUunbOIKy3QQZTXDv89yslZqi0wmeI +8sVRqSAYZRIPEylwS9CU2ReK9BJlfVLZnNP1At4gHE6S2hk8T0eVeLT8uhQiUXXJe4644UoPhoA4 +Fxgm7Q62KT6yP9O7c4eZzmQ4A9hdlWM6CtZ5pgMAzLOrVFdypzSc+S1j8DqcFkALCw83AAAAAAAA + +------=_NextPart_000_0018_01D64B14.F58791A0-- + +--===============0678627779074767862== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +HTCondor-users mailing list +To unsubscribe, send a message to htcondor-users-request@cs.wisc.edu with a +subject: Unsubscribe +You can also unsubscribe by visiting +https://lists.cs.wisc.edu/mailman/listinfo/htcondor-users + +The archives can be found at: +https://lists.cs.wisc.edu/archive/htcondor-users/ +--===============0678627779074767862==--)"; + +static std::string +message(const Regex& rx, size_t id) +{ + char buf[16]; + ::snprintf(buf, sizeof(buf), "%zu", id); + + return to_string_gchar( + g_regex_replace(rx, test_msg, -1, 0, buf, + G_REGEX_MATCH_DEFAULT, {})); +} + +struct TestData { + size_t num_maildirs; + size_t num_messages; + size_t num_threads; +}; + + +static void +setup(const TestData& tdata) +{ + /* create toplevel */ + auto top_maildir = std::string{BENCH_MAILDIRS}; + int res = g_mkdir_with_parents(top_maildir.c_str(), 0700); + g_assert_cmpuint(res,==, 0); + + /* create maildirs */ + for (size_t i = 0; i != tdata.num_maildirs; ++i) { + const auto mdir = mu_format("{}/maildir-{}", top_maildir, i); + auto res = maildir_mkdir(mdir); + g_assert(!!res); + } + const auto rx = Regex::make("@ID@"); + /* create messages */ + for (size_t n = 0; n != tdata.num_messages; ++n) { + auto mpath = mu_format("{}/maildir-{}/cur/msg-{}:2,S", + top_maildir, n % tdata.num_maildirs, + n); + std::ofstream stream(mpath); + auto msg = message(*rx, n); + stream.write(msg.c_str(), msg.size()); + g_assert_true(stream.good()); + } +} + +static void +tear_down() +{ + /* ugly */ + GError *err{}; + const auto cmd{mu_format("/bin/rm -rf '{}' '{}'", BENCH_MAILDIRS, BENCH_STORE)}; + if (!g_spawn_command_line_sync(cmd.c_str(), NULL, NULL, NULL, &err)) { + mu_warning("error: {}", err ? err->message : "?"); + g_clear_error(&err); + } +} + +void +black_hole(void) +{ + return; /* do nothing */ +} + +static void +benchmark_indexer(gconstpointer testdata) +{ + using namespace std::chrono_literals; + using Clock = std::chrono::steady_clock; + const auto tdata = reinterpret_cast<const TestData*>(testdata); + + setup(*tdata); + auto start = Clock::now(); + + { + auto store{Store::make_new(BENCH_STORE, BENCH_MAILDIRS)}; + g_assert_true(!!store); + Indexer::Config conf{}; + conf.max_threads = tdata->num_threads; + + auto res = store->indexer().start(conf); + g_assert_true(res); + while(store->indexer().is_running()) { + std::this_thread::sleep_for(100ms); + } + g_assert_cmpuint(store->size(),==, tdata->num_messages); + } + + const auto elapsed = Clock::now() - start; + std::cout << "indexed " << tdata->num_messages << " messages in " + << tdata->num_maildirs << " maildirs in " + << to_ms(elapsed) << "ms; " + << to_us(elapsed) / tdata->num_messages << " μs/message; " + << static_cast<size_t>(1000*tdata->num_messages / to_ms(elapsed)) + << " messages/s" + << " (" << tdata->num_threads << " thread(s))\n"; + + tear_down(); +} + +int +main(int argc, char *argv[]) +{ + size_t num_maildirs{}, num_messages{}; + + mu_test_init(&argc, &argv); + + if (g_test_perf()) { + num_maildirs = 20; + num_messages = 5000; + } else { + num_maildirs = 10; + num_messages = 1000; + } + + g_log_set_handler( + NULL, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION), + (GLogFunc)black_hole, + NULL); + + + size_t thread_num{}; + const auto tnum = g_getenv("THREAD_NUM"); + if (tnum) + thread_num = ::strtol(tnum, NULL, 10); + + if (thread_num != 0) { + /* THREAD_NUM specified */ + static TestData tdata{num_maildirs, num_messages, thread_num}; + char *name = g_strdup_printf("/bench/indexer/%zu-cores", thread_num); + g_test_add_data_func(name, &tdata, benchmark_indexer); + g_free(name); + } else { + /* no THREAD_NUM specified */ + + const size_t hw_threads = std::thread::hardware_concurrency(); + + { + static TestData tdata{num_maildirs, num_messages, 1}; + g_test_add_data_func("/bench/indexer/1-core", &tdata, benchmark_indexer); + } + + if (hw_threads > 2) { + static TestData tdata{num_maildirs, num_messages, hw_threads/2}; + char *name = g_strdup_printf("/bench/indexer/%zu-cores", hw_threads/2); + g_test_add_data_func(name, &tdata, benchmark_indexer); + g_free(name); + } + + if (hw_threads > 1) { + static TestData tdata{num_maildirs, num_messages, hw_threads}; + char *name = g_strdup_printf("/bench/indexer/%zu-cores", hw_threads); + g_test_add_data_func(name, &tdata, benchmark_indexer); + g_free(name); + } + } + + tear_down(); + + return g_test_run(); +} diff --git a/lib/tests/meson.build b/lib/tests/meson.build new file mode 100644 index 0000000..39b5b38 --- /dev/null +++ b/lib/tests/meson.build @@ -0,0 +1,147 @@ +## Copyright (C) 2021-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# tests +# + + +# +# unit tests +# + +test('test-threads', + executable('test-threads', + '../mu-query-threads.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) +test('test-contacts-cache', + executable('test-contacts-cache', + '../mu-contacts-cache.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-config', + executable('test-config', + '../mu-config.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-query-macros', + executable('test-query-macros', + '../mu-query-macros.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep])) + +test('test-query-processor', + executable('test-query-processor', + '../mu-query-processor.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep])) + +test('test-query-parser', + executable('test-query-parser', + '../mu-query-parser.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep])) + +test('test-query-xapianizer', + executable('test-query-xapianizer', + '../mu-query-xapianizer.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep])) + + +test('test-indexer', + executable('test-indexer', + '../mu-indexer.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, config_h_dep, + lib_mu_dep])) + +test('test-scanner', + executable('test-scanner', + '../mu-scanner.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, config_h_dep, + lib_mu_utils_dep])) + +test('test-xapian-db', + executable('test-xapian-db', + '../mu-xapian-db.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep, config_h_dep])) + +test('test-maildir', + executable('test-maildir', + 'test-mu-maildir.cc', + install: false, + dependencies: [glib_dep, lib_mu_dep])) +test('test-msg', + executable('test-msg', + 'test-mu-msg.cc', + install: false, + dependencies: [glib_dep, lib_mu_dep])) +test('test-store', + executable('test-store', + 'test-mu-store.cc', + install: false, + dependencies: [glib_dep, lib_mu_dep])) +test('test-query', + executable('test-query', + 'test-query.cc', + install: false, + dependencies: [glib_dep, gmime_dep, lib_mu_dep])) + +test('test-store-query', + executable('test-store-query', + 'test-mu-store-query.cc', + install: false, + dependencies: [glib_dep, gmime_dep, lib_mu_dep])) +# +# benchmarks +# +bench_maildirs=join_paths(meson.current_build_dir(), 'maildirs') +bench_store=join_paths(meson.current_build_dir(), 'store') +bench_indexer_exe = executable( + 'bench-indexer', + 'bench-indexer.cc', + install:false, + cpp_args:['-DBENCH_MAILDIRS="' + bench_maildirs + '"', + '-DBENCH_STORE="' + bench_store + '"', + ], + dependencies: [lib_mu_dep, glib_dep]) + +benchmark('bench-indexer', bench_indexer_exe, args: ['-m', 'perf']) + +# +# below does _not_ pass; it is believed that it's a false alarm. +# https://gitlab.gnome.org/GNOME/glib/-/issues/2662 + +# also register benchmark as a normal test so it gets included for +# valgrind/helgrind etc. +# test('test-bench-indexer', bench_indexer_exe, +# args : ['-m', 'quick'], env: ['THREADNUM=16']) diff --git a/lib/tests/test-mu-container.cc b/lib/tests/test-mu-container.cc new file mode 100644 index 0000000..4fb1939 --- /dev/null +++ b/lib/tests/test-mu-container.cc @@ -0,0 +1,80 @@ +/* +** Copyright (C) 2014 Jakub Sitnicki <jsitnicki@gmail.com> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include <glib.h> + +#include "utils/mu-test-utils.hh" +#include "mu-container.hh" + +static gboolean +container_has_children(const MuContainer* c) +{ + return c && c->child; +} + +static gboolean +container_is_sibling_of(const MuContainer* c, const MuContainer* sibling) +{ + const MuContainer* cur; + + for (cur = c; cur; cur = cur->next) { + if (cur == sibling) + return TRUE; + } + + return container_is_sibling_of(sibling, c); +} + +static void +test_mu_container_splice_children_when_parent_has_no_siblings(void) +{ + MuContainer *child, *parent, *root_set; + + child = mu_container_new(NULL, 0, "child"); + parent = mu_container_new(NULL, 0, "parent"); + parent = mu_container_append_children(parent, child); + + root_set = parent; + root_set = mu_container_splice_children(root_set, parent); + + g_assert(root_set != NULL); + g_assert(!container_has_children(parent)); + g_assert(container_is_sibling_of(root_set, child)); + + mu_container_destroy(parent); + mu_container_destroy(child); +} + +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/mu-container/mu-container-splice-children-when-parent-has-no-siblings", + test_mu_container_splice_children_when_parent_has_no_siblings); + + g_log_set_handler( + NULL, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION), + (GLogFunc)black_hole, + NULL); + + return g_test_run(); +} diff --git a/lib/tests/test-mu-maildir.cc b/lib/tests/test-mu-maildir.cc new file mode 100644 index 0000000..aee8189 --- /dev/null +++ b/lib/tests/test-mu-maildir.cc @@ -0,0 +1,557 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include <glib.h> +#include <glib/gstdio.h> + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <vector> +#include <fstream> + +#include "utils/mu-test-utils.hh" +#include "mu-maildir.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-result.hh" + +using namespace Mu; + +static void +test_maildir_mkdir_01() +{ + TempDir temp_dir; + auto mdir = join_paths(temp_dir.path(), "cuux"); + auto res{maildir_mkdir(mdir, 0755, false/*!noindex*/)}; + assert_valid_result(res); + + for (auto sub : {"tmp", "cur", "new"}) { + auto subpath = join_paths(mdir, sub); + g_assert_cmpuint(g_access(subpath.c_str(), R_OK), ==, 0); + g_assert_cmpuint(g_access(subpath.c_str(), W_OK), ==, 0); + } + + auto noindex = join_paths(mdir, ".noindex"); + g_assert_cmpuint(g_access(noindex.c_str(), F_OK), !=, 0); +} + +static void +test_maildir_mkdir_02() +{ + TempDir temp_dir; + auto mdir = join_paths(temp_dir.path(), "cuux"); + auto res{maildir_mkdir(mdir, 0755, true/*noindex*/)}; + assert_valid_result(res); + + for (auto sub : {"tmp", "cur", "new"}) { + auto subpath = join_paths(mdir, sub); + g_assert_cmpuint(g_access(subpath.c_str(), R_OK), ==, 0); + g_assert_cmpuint(g_access(subpath.c_str(), W_OK), ==, 0); + } + + auto noindex = join_paths(mdir, ".noindex"); + g_assert_cmpuint(g_access(noindex.c_str(), F_OK), ==, 0); +} + +static void +test_maildir_mkdir_03() +{ + TempDir temp_dir; + auto mdir = join_paths(temp_dir.path(), "cuux"); + + // create part already + auto curdir = join_paths(mdir, "cur"); + g_assert_cmpuint(g_mkdir_with_parents(curdir.c_str(), 0755), ==, 0); + + auto res{maildir_mkdir(mdir, 0755, false/*!noindex*/)}; + assert_valid_result(res); + + // should still work. + for (auto sub : {"tmp", "cur", "new"}) { + auto subpath = join_paths(mdir, sub); + g_assert_cmpuint(g_access(subpath.c_str(), R_OK), ==, 0); + g_assert_cmpuint(g_access(subpath.c_str(), W_OK), ==, 0); + } + + auto noindex = join_paths(mdir, ".noindex"); + g_assert_cmpuint(g_access(noindex.c_str(), F_OK), !=, 0); +} + + +static void +test_maildir_mkdir_04() +{ + allow_warnings(); + + if (geteuid() == 0) { + g_test_skip("not useful when run as root"); + return; + } + + TempDir temp_dir; + auto mdir = join_paths(temp_dir.path(), "cuux"); + g_assert_cmpuint(g_mkdir_with_parents(mdir.c_str(), 0755), ==, 0); + + auto curdir = join_paths(mdir, "cur"); + g_assert_cmpuint(g_mkdir_with_parents(curdir.c_str(), 0000), ==, 0); + + /* this should fail now, because cur is not read/writable */ + auto res = maildir_mkdir(mdir, 0755, false); + g_assert_false(!!res); +} + +static gboolean +ignore_error(const char* log_domain, GLogLevelFlags log_level, const gchar* msg, gpointer user_data) +{ + return FALSE; /* don't abort */ +} + +static void +test_maildir_mkdir_05(void) +{ + /* this must fail */ + g_test_log_set_fatal_handler((GTestLogFatalFunc)ignore_error, NULL); + + g_assert_false(!!maildir_mkdir({}, 0755, true)); +} + +[[maybe_unused]] static void +assert_matches_regexp(const char* str, const char* rx) +{ + if (!g_regex_match_simple(rx, str, (GRegexCompileFlags)0, (GRegexMatchFlags)0)) { + if (g_test_verbose()) + g_print("%s does not match %s", str, rx); + g_assert(0); + } +} + + +static void +test_determine_target_ok(void) +{ + struct TestCase { + std::string old_path; + std::string root_maildir; + std::string target_maildir; + Flags new_flags; + bool new_name; + std::string expected; + }; + const std::vector<TestCase> testcases = { + TestCase{ /* change some flags */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "/home/foo/Maildir", + {}, + Flags::Seen | Flags::Passed, + false, + "/home/foo/Maildir/test/cur/123456:2,PS" + }, + + TestCase{ /* from cur -> new */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "/home/foo/Maildir", + {}, + Flags::New, + false, + "/home/foo/Maildir/test/new/123456" + }, + + TestCase{ /* from new->cur */ + "/home/foo/Maildir/test/cur/123456", + "/home/foo/Maildir", + {}, + Flags::Seen | Flags::Flagged, + false, + "/home/foo/Maildir/test/cur/123456:2,FS" + }, + + TestCase{ /* change maildir */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "/home/foo/Maildir", + "/test2", + Flags::Flagged | Flags::Replied, + false, + "/home/foo/Maildir/test2/cur/123456:2,FR" + }, + TestCase{ /* remove all flags */ + "/home/foo/Maildir/test/new/123456", + "/home/foo/Maildir", + {}, + Flags::None, + false, + "/home/foo/Maildir/test/cur/123456:2," + }, + }; + + for (auto&& testcase: testcases) { + const auto res = maildir_determine_target( + testcase.old_path, + testcase.root_maildir, + testcase.target_maildir, + testcase.new_flags, + testcase.new_name); + g_assert_true(!!res); + g_assert_cmpstr(testcase.expected.c_str(), ==, + res.value().c_str()); + } +} + + + +static void +test_determine_target_fail(void) +{ + struct TestCase { + std::string old_path; + std::string root_maildir; + std::string target_maildir; + Flags new_flags; + bool new_name; + std::string expected; + }; + const std::vector<TestCase> testcases = { + TestCase{ /* fail: no absolute path */ + "../foo/Maildir/test/cur/123456:2,FR-not-absolute", + "/home/foo/Maildir", + {}, + Flags::Seen | Flags::Passed, + false, + "/home/foo/Maildir/test/cur/123456:2,PS" + }, + + TestCase{ /* fail: no absolute root */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "../foo/Maildir-not-absolute", + {}, + Flags::New, + false, + "/home/foo/Maildir/test/new/123456" + }, + + TestCase{ /* fail: maildir must start with '/' */ + "/home/foo/Maildir/test/cur/123456", + "/home/foo/Maildir", + "mymaildirwithoutslash", + Flags::Seen | Flags::Flagged, + false, + "/home/foo/Maildir/test/cur/123456:2,FS" + }, + + TestCase{ /* fail: path must be below maildir */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "/home/bar/Maildir", + "/test2", + Flags::Flagged | Flags::Replied, + false, + "/home/foo/Maildir/test2/cur/123456:2,FR" + }, + TestCase{ /* fail: New cannot be combined */ + "/home/foo/Maildir/test/new/123456", + "/home/foo/Maildir", + {}, + Flags::New | Flags::Replied, + false, + "/home/foo/Maildir/test/cur/123456:2," + }, + }; + + for (auto&& testcase: testcases) { + const auto res = maildir_determine_target( + testcase.old_path, + testcase.root_maildir, + testcase.target_maildir, + testcase.new_flags, + testcase.new_name); + g_assert_false(!!res); + } +} + + + +static void +test_maildir_get_new_path_01(void) +{ + struct { + std::string oldpath; + Flags flags; + std::string newpath; + } paths[] = {{"/home/foo/Maildir/test/cur/123456:2,FR", + Flags::Replied, + "/home/foo/Maildir/test/cur/123456:2,R"}, + {"/home/foo/Maildir/test/cur/123456:2,FR", + Flags::New, + "/home/foo/Maildir/test/new/123456"}, + {"/home/foo/Maildir/test/new/123456:2,FR", + (Flags::Seen | Flags::Replied), + "/home/foo/Maildir/test/cur/123456:2,RS"}, + {"/home/foo/Maildir/test/new/1313038887_0.697", + (Flags::Seen | Flags::Flagged | Flags::Passed), + "/home/foo/Maildir/test/cur/1313038887_0.697:2,FPS"}, + {"/home/foo/Maildir/test/new/1313038887_0.697:2,", + (Flags::Seen | Flags::Flagged | Flags::Passed), + "/home/foo/Maildir/test/cur/1313038887_0.697:2,FPS"}, + /* note the ':2,' suffix on the new message is + * removed */ + + {"/home/foo/Maildir/trash/new/1312920597.2206_16.cthulhu", + Flags::Seen, + "/home/foo/Maildir/trash/cur/1312920597.2206_16.cthulhu:2,S"}}; + + for (int i = 0; i != G_N_ELEMENTS(paths); ++i) { + const auto newpath{maildir_determine_target(paths[i].oldpath, + "/home/foo/Maildir", + {}, paths[i].flags, false)}; + assert_valid_result(newpath); + assert_equal(*newpath, paths[i].newpath); + } +} + +static void +test_maildir_get_new_path_02(void) +{ + struct { + std::string oldpath; + Flags flags; + std::string targetdir; + std::string newpath; + std::string root_maildir; + } paths[] = {{"/home/foo/Maildir/test/cur/123456:2,FR", + Flags::Replied, + "/blabla", + "/home/foo/Maildir/blabla/cur/123456:2,R", + "/home/foo/Maildir"}, + {"/home/bar/Maildir/test/cur/123456:2,FR", + Flags::New, + "/coffee", + "/home/bar/Maildir/coffee/new/123456", + "/home/bar/Maildir" + }, + {"/home/cuux/Maildir/test/new/123456", + (Flags::Seen | Flags::Replied), + "/tea", + "/home/cuux/Maildir/tea/cur/123456:2,RS", + "/home/cuux/Maildir"}, + {"/home/boy/Maildir/test/new/1313038887_0.697:2,", + (Flags::Seen | Flags::Flagged | Flags::Passed), + "/stuff", + "/home/boy/Maildir/stuff/cur/1313038887_0.697:2,FPS", + "/home/boy/Maildir"}}; + + for (int i = 0; i != G_N_ELEMENTS(paths); ++i) { + auto newpath{maildir_determine_target(paths[i].oldpath, + paths[i].root_maildir, + paths[i].targetdir, + paths[i].flags, + false)}; + assert_valid_result(newpath); + assert_equal(*newpath, paths[i].newpath); + } +} + + +static void +test_maildir_get_new_path_custom_real(bool change_name) +{ + struct { + std::string oldpath; + Flags flags; + std::string targetdir; + std::string newpath; + std::string root_maildir; + } paths[] = {{"/home/foo/Maildir/test/cur/123456:2,FR", + Flags::Replied, + "/blabla", + "/home/foo/Maildir/blabla/cur/123456:2,R", + "/home/foo/Maildir"}, + {"/home/foo/Maildir/test/cur/123456:2,hFeRllo123", + Flags::Flagged, + "/blabla", + "/home/foo/Maildir/blabla/cur/123456:2,F", + "/home/foo/Maildir"}, + {"/home/foo/Maildir/test/cur/123456:2,abc", + Flags::Passed, + "/blabla", + "/home/foo/Maildir/blabla/cur/123456:2,P", + "/home/foo/Maildir"}}; + + for (int i = 0; i != G_N_ELEMENTS(paths); ++i) { + auto newpath{maildir_determine_target(paths[i].oldpath, + paths[1].root_maildir, + paths[i].targetdir, + paths[i].flags, + change_name)}; + assert_valid_result(newpath); + if (change_name) + g_assert_true(*newpath != paths[i].newpath); // weak test + else + assert_equal(*newpath, paths[i].newpath); + } +} + + +static void +test_maildir_get_new_path_custom(void) +{ + return test_maildir_get_new_path_custom_real(false); +} + + +static void +test_maildir_get_new_path_custom_change_name(void) +{ + return test_maildir_get_new_path_custom_real(true); +} + + +static void +test_maildir_from_path(void) +{ + unsigned u; + + struct { + std::string path, exp; + } cases[] = {{"/home/foo/Maildir/test/cur/123456:2,FR", "/test"}, + {"/home/foo/Maildir/lala/new/1313038887_0.697:2,", "/lala"}}; + + for (u = 0; u != G_N_ELEMENTS(cases); ++u) { + auto mdir{maildir_from_path(cases[u].path, "/home/foo/Maildir")}; + assert_valid_result(mdir); + assert_equal(*mdir, cases[u].exp); + } +} + +static void +test_maildir_link() +{ + TempDir tmpdir; + + assert_valid_result(maildir_mkdir(tmpdir.path() + "/foo")); + assert_valid_result(maildir_mkdir(tmpdir.path() + "/bar")); + + const auto srcpath1 = tmpdir.path() + "/foo/cur/msg1"; + const auto srcpath2 = tmpdir.path() + "/foo/new/msg2"; + + { + std::ofstream stream(srcpath1); + stream.write("cur", 3); + g_assert_true(stream.good()); + stream.close(); + } + + { + std::ofstream stream(srcpath2); + stream.write("new", 3); + g_assert_true(stream.good()); + stream.close(); + } + + assert_valid_result(maildir_link(srcpath1, tmpdir.path() + "/bar", false)); + assert_valid_result(maildir_link(srcpath2, tmpdir.path() + "/bar", false)); + + const auto dstpath1 = tmpdir.path() + "/bar/cur/msg1"; + const auto dstpath2 = tmpdir.path() + "/bar/new/msg2"; + + g_assert_true(g_access(dstpath1.c_str(), F_OK) == 0); + g_assert_true(g_access(dstpath2.c_str(), F_OK) == 0); + + g_assert_false(!!maildir_clear_links("/nonexistent/bla/foo/xuux")); + + assert_valid_result(maildir_clear_links(tmpdir.path() + "/bar")); + g_assert_false(g_access(dstpath1.c_str(), F_OK) == 0); + g_assert_false(g_access(dstpath2.c_str(), F_OK) == 0); +} + + +static void +test_maildir_move(bool assume_remote) +{ + TempDir tmpdir; + + assert_valid_result(maildir_mkdir(tmpdir.path() + "/foo")); + assert_valid_result(maildir_mkdir(tmpdir.path() + "/bar")); + + const auto srcpath1{join_paths(tmpdir.path(), "/foo/cur/msg1")}; + const auto srcpath2{join_paths(tmpdir.path(), "/foo/new/msg2")}; + + { + std::ofstream stream(srcpath1); + stream.write("cur", 3); + g_assert_true(stream.good()); + stream.close(); + } + + { + std::ofstream stream(srcpath2); + stream.write("new", 3); + g_assert_true(stream.good()); + stream.close(); + } + + const auto dstpath = tmpdir.path() + "/test1"; + + assert_valid_result(maildir_move_message(srcpath1, dstpath, assume_remote)); + assert_valid_result(maildir_move_message(srcpath2, dstpath, assume_remote)); + + assert_valid_result(maildir_move_message(dstpath, dstpath)); // self-move is okay. +} + +static void +test_maildir_move_vanilla() +{ + test_maildir_move(false/*!assume_remote*/); +} + +static void +test_maildir_move_remote() +{ + test_maildir_move(true/*assume_remote*/); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + /* mu_util_maildir_mkmdir */ + g_test_add_func("/maildir/mkdir-01", test_maildir_mkdir_01); + g_test_add_func("/maildir/mkdir-02", test_maildir_mkdir_02); + g_test_add_func("/maildir/mkdir-03", test_maildir_mkdir_03); + g_test_add_func("/maildir/mkdir-04", test_maildir_mkdir_04); + g_test_add_func("/maildir/mkdir-05", test_maildir_mkdir_05); + + g_test_add_func("/maildir/determine-target-ok", test_determine_target_ok); + g_test_add_func("/maildir/determine-target-fail", test_determine_target_fail); + + // /* get/set flags */ + g_test_add_func("/maildir/get-new-path-01", test_maildir_get_new_path_01); + g_test_add_func("/maildir/get-new-path-02", test_maildir_get_new_path_02); + g_test_add_func("/maildir/get-new-path-custom", test_maildir_get_new_path_custom); + g_test_add_func("/maildir/get-new-path-custom-change-name", + test_maildir_get_new_path_custom_change_name); + + g_test_add_func("/maildir/from-path", test_maildir_from_path); + + g_test_add_func("/maildir/link", test_maildir_link); + g_test_add_func("/maildir/move-vanilla", test_maildir_move_vanilla); + g_test_add_func("/maildir/move-remote", test_maildir_move_remote); + + return g_test_run(); +} diff --git a/lib/tests/test-mu-msg-fields.cc b/lib/tests/test-mu-msg-fields.cc new file mode 100644 index 0000000..5f5df16 --- /dev/null +++ b/lib/tests/test-mu-msg-fields.cc @@ -0,0 +1,126 @@ +/* +** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif /*HAVE_CONFIG_H*/ + +#include <glib.h> +#include <stdlib.h> +#include <unistd.h> +#include <time.h> + +#include <locale.h> + +#include "utils/mu-test-utils.hh" +#include "mu-message-fields.hh" + +static void +test_mu_msg_field_body(void) +{ + Field::Id field; + + field = Field::Id::BodyText; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "body"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 'b'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'B'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, FALSE); +} + +static void +test_mu_msg_field_subject(void) +{ + Field::Id field; + + field = Field::Id::Subject; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "subject"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 's'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'S'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, FALSE); +} + +static void +test_mu_msg_field_to(void) +{ + Field::Id field; + + field = Field::Id::To; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "to"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 't'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'T'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, FALSE); +} + +static void +test_mu_msg_field_prio(void) +{ + Field::Id field; + + field = Field::Id::Priority; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "prio"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 'p'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'P'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, TRUE); +} + +static void +test_mu_msg_field_flags(void) +{ + Field::Id field; + + field = Field::Id::Flags; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "flag"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 'g'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'G'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, TRUE); +} + +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + /* mu_msg_str_date */ + g_test_add_func("/mu-msg-fields/mu-msg-field-body", test_mu_msg_field_body); + g_test_add_func("/mu-msg-fields/mu-msg-field-subject", test_mu_msg_field_subject); + g_test_add_func("/mu-msg-fields/mu-msg-field-to", test_mu_msg_field_to); + g_test_add_func("/mu-msg-fields/mu-msg-field-prio", test_mu_msg_field_prio); + g_test_add_func("/mu-msg-fields/mu-msg-field-flags", test_mu_msg_field_flags); + + /* FIXME: add tests for mu_msg_str_flags; but note the + * function simply calls mu_msg_field_str */ + + g_log_set_handler( + NULL, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION), + (GLogFunc)black_hole, + NULL); + + return g_test_run(); +} diff --git a/lib/tests/test-mu-msg.cc b/lib/tests/test-mu-msg.cc new file mode 100644 index 0000000..1e5d82d --- /dev/null +++ b/lib/tests/test-mu-msg.cc @@ -0,0 +1,355 @@ +/* +** Copyright (C) 2008-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <glib.h> +#include <stdlib.h> +#include <unistd.h> +#include <time.h> +#include <array> +#include <string> + +#include <locale.h> + +#include "utils/mu-test-utils.hh" +#include "utils/mu-result.hh" +#include "utils/mu-utils.hh" + +#include <message/mu-message.hh> + +using namespace Mu; + +using ExpectedContacts = const std::vector<std::pair<std::string, std::string>>; + +static void +assert_contacts_equal(const Contacts& contacts, + const ExpectedContacts& expected) +{ + g_assert_cmpuint(contacts.size(), ==, expected.size()); + + size_t n{}; + for (auto&& contact: contacts) { + if (g_test_verbose()) + mu_message("{{ \"{}\", \"{}\"}},\n", + contact.name, contact.email); + assert_equal(contact.name, expected.at(n).first); + assert_equal(contact.email, expected.at(n).second); + ++n; + } + mu_print("\n"); +} + + +static void +test_mu_msg_01(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1220863042.12663_1.mindcrime!2,S") + .value()}; + + assert_contacts_equal(msg.to(), {{ "Donald Duck", "gcc-help@gcc.gnu.org" }}); + assert_contacts_equal(msg.from(), {{ "Mickey Mouse", "anon@example.com" }}); + + assert_equal(msg.subject(), "gcc include search order"); + assert_equal(msg.message_id(), + "3BE9E6535E3029448670913581E7A1A20D852173@" + "emss35m06.us.lmco.com"); + assert_equal(msg.header("Mailing-List").value_or(""), + "contact gcc-help-help@gcc.gnu.org; run by ezmlm"); + g_assert_true(msg.priority() == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 1217530645); + + assert_contacts_equal(msg.all_contacts(), { + { "", "gcc-help-owner@gcc.gnu.org"}, + { "Mickey Mouse", "anon@example.com" }, + { "Donald Duck", "gcc-help@gcc.gnu.org" } + }); + +} + +static void +test_mu_msg_02(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1220863087.12663_19.mindcrime!2,S") + .value()}; + + assert_equal(msg.to().at(0).email, "help-gnu-emacs@gnu.org"); + assert_equal(msg.subject(), "Re: Learning LISP; Scheme vs elisp."); + assert_equal(msg.from().at(0).email, "anon@example.com"); + assert_equal(msg.message_id(), "r6bpm5-6n6.ln1@news.ducksburg.com"); + assert_equal(msg.header("Errors-To").value_or(""), + "help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org"); + g_assert_true(msg.priority() /* 'low' */ + == Priority::Low); + g_assert_cmpuint(msg.date(), ==, 1218051515); + mu_println("flags: {}", Mu::to_string(msg.flags())); + g_assert_true(msg.flags() == (Flags::Seen|Flags::MailingList)); + + assert_contacts_equal(msg.all_contacts(), { + { "", "help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org"}, + { "", "anon@example.com"}, + { "", "help-gnu-emacs@gnu.org"}, + }); + +} + +static void +test_mu_msg_03(void) +{ + //const GSList* params; + + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1283599333.1840_11.cthulhu!2,") + .value()}; + + assert_equal(msg.to().at(0).display_name(), "Bilbo Baggins <bilbo@anotherexample.com>"); + assert_equal(msg.subject(), "Greetings from Lothlórien"); + assert_equal(msg.from().at(0).display_name(), "Frodo Baggins <frodo@example.com>"); + g_assert_true(msg.priority() == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 0); + assert_equal(msg.body_text().value_or(""), + "\nLet's write some fünkÿ text\nusing umlauts.\n\nFoo.\n"); + + // params = mu_msg_get_body_text_content_type_parameters(msg, MU_MSG_OPTION_NONE); + // g_assert_cmpuint(g_slist_length((GSList*)params), ==, 2); + + // assert_equal((char*)params->data, "charset"); + // params = g_slist_next(params); + // assert_equal((char*)params->data, "UTF-8"); + g_assert_true(msg.flags() == (Flags::Unread)); +} + +static void +test_mu_msg_04(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/mail5").value()}; + + assert_equal(msg.to().at(0).display_name(), "George Custer <gac@example.com>"); + assert_equal(msg.subject(), "pics for you"); + assert_equal(msg.from().at(0).display_name(), "Sitting Bull <sb@example.com>"); + g_assert_true(msg.priority() /* 'low' */ + == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 0); + g_assert_true(msg.flags() == + (Flags::HasAttachment|Flags::Unread)); + g_assert_true(msg.flags() == + (Flags::HasAttachment|Flags::Unread)); +} + +static void +test_mu_msg_multimime(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/multimime!2,FS").value()}; + + /* ie., are text parts properly concatenated? */ + assert_equal(msg.subject(), "multimime"); + assert_equal(msg.body_text().value_or(""), "abcdef"); + g_assert_true(msg.flags() == (Flags::HasAttachment|Flags::Flagged|Flags::Seen)); +} + +static void +test_mu_msg_flags(void) +{ + std::array<std::pair<std::string, Flags>, 2> tests= {{ + {MU_TESTMAILDIR4 "/multimime!2,FS", + (Flags::Flagged | Flags::Seen | + Flags::HasAttachment)}, + {MU_TESTMAILDIR4 "/special!2,Sabc", + (Flags::Seen)} + }}; + + for (auto&& test: tests) { + auto msg = Message::make_from_path(test.first); + assert_valid_result(msg); + g_assert_true(msg->flags() == test.second); + } +} + +static void +test_mu_msg_umlaut(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1305664394.2171_402.cthulhu!2,") + .value()}; + + assert_contacts_equal(msg.to(), { { "Helmut Kröger", "hk@testmu.xxx"}}); + assert_contacts_equal(msg.from(), { { "Mü", "testmu@testmu.xx"}}); + + assert_equal(msg.subject(), "Motörhead"); + assert_equal(msg.from().at(0).display_name(), "Mü <testmu@testmu.xx>"); + g_assert_true(msg.priority() == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 0); +} + +static void +test_mu_msg_references(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1305664394.2171_402.cthulhu!2,") + .value()}; + + std::array<std::string, 4> expected_refs = { + "non-exist-01@msg.id", + "non-exist-02@msg.id", + "non-exist-03@msg.id", + "non-exist-04@msg.id" + }; + + assert_equal_seq_str(msg.references(), expected_refs); + assert_equal(msg.thread_id(), expected_refs[0]); +} + +static void +test_mu_msg_references_dups(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1252168370_3.14675.cthulhu!2,S") + .value()}; + + std::array<std::string, 6> expected_refs = { + "439C1136.90504@euler.org", + "4399DD94.5070309@euler.org", + "20051209233303.GA13812@gauss.org", + "439B41ED.2080402@euler.org", + "439A1E03.3090604@euler.org", + "20051211184308.GB13513@gauss.org" + }; + + assert_equal_seq_str(msg.references(), expected_refs); + assert_equal(msg.thread_id(), expected_refs[0]); +} + +static void +test_mu_msg_references_many(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR2 "/bar/cur/181736.eml") + .value()}; + + std::array<std::string, 11> expected_refs = { + "e9065dac-13c1-4103-9e31-6974ca232a89@t15g2000prt.googlegroups.com", + "87hbblwelr.fsf@sapphire.mobileactivedefense.com", + "pql248-4va.ln1@wilbur.25thandClement.com", + "ikns6r$li3$1@Iltempo.Update.UU.SE", + "8762s0jreh.fsf@sapphire.mobileactivedefense.com", + "ikqqp1$jv0$1@Iltempo.Update.UU.SE", + "87hbbjc5jt.fsf@sapphire.mobileactivedefense.com", + "ikr0na$lru$1@Iltempo.Update.UU.SE", + "tO8cp.1228$GE6.370@news.usenetserver.com", + "ikr6ks$nlf$1@Iltempo.Update.UU.SE", + "8ioh48-8mu.ln1@leafnode-msgid.gclare.org.uk" + }; + + assert_equal_seq_str(msg.references(), expected_refs); + assert_equal(msg.thread_id(), expected_refs[0]); +} + +static void +test_mu_msg_tags(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/mail1").value()}; + + assert_contacts_equal(msg.to(), {{ "Julius Caesar", "jc@example.com" }}); + assert_contacts_equal(msg.from(), {{ "John Milton", "jm@example.com" }}); + + assert_equal(msg.subject(),"Fere libenter homines id quod volunt credunt"); + + g_assert_true(msg.priority() == Priority::High); + g_assert_cmpuint(msg.date(), ==, 1217530645); + + std::array<std::string, 4> expected_tags = { + "Paradise", + "losT", + "john", + "milton" + }; + assert_equal_seq_str(msg.tags(), expected_tags); +} + +static void +test_mu_msg_comp_unix_programmer(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/181736.eml").value()}; + + g_assert_true(msg.to().empty()); + assert_equal(msg.subject(), + "Re: Are writes \"atomic\" to readers of the file?"); + assert_equal(msg.from().at(0).display_name(), "Jimbo Foobarcuux <jimbo@slp53.sl.home>"); + assert_equal(msg.message_id(), "oktdp.42997$Te.22361@news.usenetserver.com"); + + auto refs = join(msg.references(), ','); + assert_equal(refs, + "e9065dac-13c1-4103-9e31-6974ca232a89@t15g2000prt" + ".googlegroups.com," + "87hbblwelr.fsf@sapphire.mobileactivedefense.com," + "pql248-4va.ln1@wilbur.25thandClement.com," + "ikns6r$li3$1@Iltempo.Update.UU.SE," + "8762s0jreh.fsf@sapphire.mobileactivedefense.com," + "ikqqp1$jv0$1@Iltempo.Update.UU.SE," + "87hbbjc5jt.fsf@sapphire.mobileactivedefense.com," + "ikr0na$lru$1@Iltempo.Update.UU.SE," + "tO8cp.1228$GE6.370@news.usenetserver.com," + "ikr6ks$nlf$1@Iltempo.Update.UU.SE," + "8ioh48-8mu.ln1@leafnode-msgid.gclare.org.uk"); + + //"jimbo@slp53.sl.home (Jimbo Foobarcuux)"; + g_assert_true(msg.priority() == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 1299603860); +} + +static void +test_mu_str_prio_01(void) +{ + g_assert_true(priority_name(Priority::Low) == "low"); + g_assert_true(priority_name(Priority::Normal) == "normal"); + g_assert_true(priority_name(Priority::High) == "high"); +} + +G_GNUC_UNUSED static gboolean +ignore_error(const char* log_domain, GLogLevelFlags log_level, const gchar* msg, gpointer user_data) +{ + return FALSE; /* don't abort */ +} + + +int +main(int argc, char* argv[]) +{ + int rv; + + g_test_init(&argc, &argv, NULL); + + /* mu_msg_str_date */ + g_test_add_func("/mu-msg/mu-msg-01", test_mu_msg_01); + g_test_add_func("/mu-msg/mu-msg-02", test_mu_msg_02); + g_test_add_func("/mu-msg/mu-msg-03", test_mu_msg_03); + g_test_add_func("/mu-msg/mu-msg-04", test_mu_msg_04); + g_test_add_func("/mu-msg/mu-msg-multimime", test_mu_msg_multimime); + + g_test_add_func("/mu-msg/mu-msg-flags", test_mu_msg_flags); + + g_test_add_func("/mu-msg/mu-msg-tags", test_mu_msg_tags); + g_test_add_func("/mu-msg/mu-msg-references", test_mu_msg_references); + g_test_add_func("/mu-msg/mu-msg-references_dups", test_mu_msg_references_dups); + g_test_add_func("/mu-msg/mu-msg-references_many", test_mu_msg_references_many); + + g_test_add_func("/mu-msg/mu-msg-umlaut", test_mu_msg_umlaut); + g_test_add_func("/mu-msg/mu-msg-comp-unix-programmer", test_mu_msg_comp_unix_programmer); + + g_test_add_func("/mu-str/mu-str-prio-01", test_mu_str_prio_01); + + rv = g_test_run(); + + return rv; +} diff --git a/lib/tests/test-mu-store-query.cc b/lib/tests/test-mu-store-query.cc new file mode 100644 index 0000000..5f28286 --- /dev/null +++ b/lib/tests/test-mu-store-query.cc @@ -0,0 +1,913 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "utils/mu-result.hh" +#include <array> +#include <thread> +#include <string> +#include <string_view> +#include <fstream> +#include <unordered_map> + +#include <mu-store.hh> +#include <mu-maildir.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-utils-file.hh> +#include <utils/mu-test-utils.hh> +#include <message/mu-message.hh> + +#include "mu-query-parser.hh" + +using namespace Mu; + + +/// map of some (unique) path-tail to the message-text +using TestMap = std::unordered_map<std::string, std::string>; + +static Store +make_test_store(const std::string& test_path, const TestMap& test_map, + Option<const Config&> conf={}) +{ + const auto maildir{join_paths(test_path, "/Maildir/")}; + // note the trailing '/' + g_test_bug("2513"); + + /* write messages to disk */ + for (auto&& item: test_map) { + + /* create the directory for the message */ + const auto msgpath{join_paths(maildir, item.first)}; + auto dir = to_string_gchar(g_path_get_dirname(msgpath.c_str())); + if (g_test_verbose()) + mu_message("create maildir {}", dir.c_str()); + + g_assert_cmpuint(g_mkdir_with_parents(dir.c_str(), 0700), ==, 0); + + /* write the file */ + std::ofstream stream(msgpath); + stream.write(item.second.data(), item.second.size()); + g_assert_true(stream.good()); + stream.close(); + } + + auto store = Store::make_new(test_path, maildir, conf); + assert_valid_result(store); + + /* index the messages */ + g_assert_true(store->indexer().start({},true/*block*/)); + if (test_map.size() > 0) + g_assert_false(store->empty()); + + g_assert_cmpuint(store->size(),==,test_map.size()); + + /* and we have a fully-ready store */ + return std::move(store.value()); +} + +static void +test_simple() +{ + const TestMap test_msgs = {{ + +// "sqlite-msg" "Simple mailing list message. +{ +"basic/cur/sqlite-msg:2,S", +R"(Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 32F276963F + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:34 +0300 (EEST) +Message-Id: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: "Foo Example" <foo@example.com> +To: sqlite-dev@sqlite.org +Cc: "Bank of America" <bank@example.com> +Bcc: Aku Ankka <donald.duck@duckstad.nl> +Mime-Version: 1.0 (Apple Message framework v926) +Date: Mon, 4 Aug 2008 11:40:49 +0200 +X-Mailer: Apple Mail (2.926) +Subject: [sqlite-dev] VM optimization inside sqlite3VdbeExec +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Sender: sqlite-dev-bounces@sqlite.org + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; + +I said: "Aujourd'hui!" +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + + // matches + for (auto&& expr: { + "Inside", + "from:foo@example.com", + "from:Foo", + "from:\"Foo Example\"", + "from:/Foo.*Example/", + "recip:\"Bank Of America\"", + "cc:bank@example.com", + "cc:bank", + "cc:america", + "bcc:donald.duck@duckstad.nl", + "bcc:donald.duck", + "bcc:duckstad.nl", + "bcc:aku", + "bcc:ankka", + "bcc:\"aku ankka\"", + "date:2008-08-01..2008-09-01", + "prio:low", + "to:sqlite-dev@sqlite.org", + "list:sqlite-dev.sqlite.org", + "aujourd'hui", +#ifdef HAVE_CLD2 + "lang:en", +#endif /*HAVE_CLD2*/ + }) { + + if (g_test_verbose()) + mu_message("query: '{}'\n", expr, + make_xapian_query(store, expr)->get_description()); + + auto qr = store.run_query(expr); + assert_valid_result(qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + } + + auto qr = store.run_query("statement"); + assert_valid_result(qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + + assert_equal(qr->begin().subject().value_or(""), + "[sqlite-dev] VM optimization inside sqlite3VdbeExec"); + g_assert_true(qr->begin().references().empty()); + //g_assert_cmpuint(qr->begin().date().value_or(0), ==, 123454); +} + +static void +test_spam_address_components() +{ + const TestMap test_msgs = {{ + +// "sqlite-msg" "Simple mailing list message. +{ +"spam/cur/spam-msg:2,S", +R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +To: example@example.com +Subject: ***SPAM*** this is a test + +Boo! +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + + g_test_bug("2278"); + g_test_bug("2281"); + + // matches both + for (auto&& expr: { + "SPAM", + "spam", + "/.*SPAM.*/", + "subject:SPAM", + "from:bar@example.com", + "subject:\\*\\*\\*SPAM\\*\\*\\*", + "bar", + "example.com" + }) { + + if (g_test_verbose()) + g_message("query: '%s'", expr); + auto qr = store.run_query(expr); + assert_valid_result(qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + } +} + + +static void +test_dups_related() +{ + const TestMap test_msgs = {{ +/* parent */ +{ +"inbox/cur/msg1:2,S", +R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Sat, 06 Aug 2022 11:01:54 -0700 +To: example@example.com +Subject: test1 + +Parent +)"}, +/* child (dup vv) */ +{ +"boo/cur/msg2:1,S", +R"(Message-Id: <edcba@foo.bar> +In-Reply-To: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Sat, 06 Aug 2022 13:01:54 -0700 +To: example@example.com +Subject: Re: test1 + +Child +)"}, +/* child (dup ^^) */ +{ +"inbox/cur/msg2:1,S", +R"(Message-Id: <edcba@foo.bar> +In-Reply-To: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Sat, 06 Aug 2022 14:01:54 -0700 +To: example@example.com +Subject: Re: test1 + +Child +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + { + // direct matches + auto qr = store.run_query("test1", Field::Id::Date, + QueryFlags::None); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 3); + } + + { + // skip duplicate messages; which one is skipped is arbitrary. + auto qr = store.run_query("test1", Field::Id::Date, + QueryFlags::SkipDuplicates); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 2); + } + + { + // no related + auto qr = store.run_query("Parent", Field::Id::Date); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + } + + { + // find related messages + auto qr = store.run_query("Parent", Field::Id::Date, + QueryFlags::IncludeRelated); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 3); + } + + { + // find related messages, skip dups. the leader message + // should _not_ be skipped. + auto qr = store.run_query("test1 AND maildir:/inbox", + Field::Id::Date, + QueryFlags::IncludeRelated| + QueryFlags::SkipDuplicates); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 2); + + // ie the /boo is to be skipped, since it's not in the leader + // set. + for (auto&& m: *qr) + assert_equal(m.message()->maildir(), "/inbox"); + } + + { + // find related messages, find parent from child. + auto qr = store.run_query("Child and maildir:/inbox", + Field::Id::Date, + QueryFlags::IncludeRelated); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 3); + + } + + { + // find related messages, find parent from child. + // leader message wins + auto qr = store.run_query("Child and maildir:/inbox", + Field::Id::Date, + QueryFlags::IncludeRelated| + QueryFlags::SkipDuplicates| + QueryFlags::Descending); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 2); + + // ie the /boo is to be skipped, since it's not in the leader + // set. + for (auto&& m: *qr) + assert_equal(m.message()->maildir(), "/inbox"); + } +} + + +static void +test_related_missing_root() +{ + const TestMap test_msgs = {{ +{ +"inbox/cur/msg1:2,S", +R"(Content-Type: text/plain; charset=utf-8 +References: <EZrZOnVCsYfFcX3Ls0VFoRnJdCGV4GM5YtO739l-iOB2ADNH7cIJWb0DaO5Of3BWDUEKq18Rz3a7rNoI96bNwQ==@protonmail.internalid> +To: "Joerg Roedel" <joro@8bytes.org>, "Suman Anna" <s-anna@ti.com> +Reply-To: "Dan Carpenter" <dan.carpenter@oracle.com> +From: "Dan Carpenter" <dan.carpenter@oracle.com> +Subject: [PATCH] iommu/omap: fix buffer overflow in debugfs +Date: Thu, 4 Aug 2022 17:32:39 +0300 +Message-Id: <YuvYh1JbE3v+abd5@kili> +List-Id: <kernel-janitors.vger.kernel.org> +Precedence: bulk + +There are two issues here: +)"}, +{ +"inbox/cur/msg2:2,S", +R"(Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=utf-8 +References: <YuvYh1JbE3v+abd5@kili> + <9pEUi_xoxa7NskF7EK_qfrlgjXzGsyw9K7cMfYbo-KI6fnyVMKTpc8E2Fu94V8xedd7cMpn0LlBrr9klBMflpw==@protonmail.internalid> +Reply-To: "Laurent Pinchart" <laurent.pinchart@ideasonboard.com> +From: "Laurent Pinchart" <laurent.pinchart@ideasonboard.com> +Subject: Re: [PATCH] iommu/omap: fix buffer overflow in debugfs +List-Id: <kernel-janitors.vger.kernel.org> +Message-Id: <YuvzKJM66k+ZPD9c@pendragon.ideasonboard.com> +Precedence: bulk +In-Reply-To: <YuvYh1JbE3v+abd5@kili> + +Hi Dan, + +Thank you for the patch. +)"}, +{ +"inbox/cur/msg3:2,S", +R"(Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=utf-8 +References: <YuvYh1JbE3v+abd5@kili> + <G6TStg8J52Q-uSMTR7wRQdPeloxpZMiEQT_F8_JIDYM25eEPeHGgrNKO0fuO78MiQgD9Mz4BDtsZlZgmPKFe4Q==@protonmail.internalid> +To: "Dan Carpenter" <dan.carpenter@oracle.com>, "Joerg Roedel" + <joro@8bytes.org>, "Suman Anna" <s-anna@ti.com> +Reply-To: "Robin Murphy" <robin.murphy@arm.com> +From: "Robin Murphy" <robin.murphy@arm.com> +Subject: Re: [PATCH] iommu/omap: fix buffer overflow in debugfs +List-Id: <kernel-janitors.vger.kernel.org> +Message-Id: <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> +Precedence: bulk +In-Reply-To: <YuvYh1JbE3v+abd5@kili> +Date: Thu, 4 Aug 2022 17:31:39 +0100 + +On 04/08/2022 3:32 pm, Dan Carpenter wrote: +> There are two issues here: +)"}, +{ +"inbox/new/msg4", +R"(Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=utf-8 +References: <YuvYh1JbE3v+abd5@kili> + <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> + <T4CDWjUrgtI5n4mh1JEdW6RLYzqbPE9-yDrhEVwDM22WX-198fBwcnLd-4_xR1gvsVSHQps9fp_pZevTF0ZmaA==@protonmail.internalid> +To: "Robin Murphy" <robin.murphy@arm.com> +Reply-To: "Dan Carpenter" <dan.carpenter@oracle.com> +From: "Dan Carpenter" <dan.carpenter@oracle.com> +Subject: Re: [PATCH] iommu/omap: fix buffer overflow in debugfs +List-Id: <kernel-janitors.vger.kernel.org> +Date: Fri, 5 Aug 2022 09:37:02 +0300 +In-Reply-To: <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> +Precedence: bulk +Message-Id: <20220805063702.GH3438@kadam> + +On Thu, Aug 04, 2022 at 05:31:39PM +0100, Robin Murphy wrote: +> On 04/08/2022 3:32 pm, Dan Carpenter wrote: +> > There are two issues here: +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + { + auto qr = store.run_query("fix buffer overflow in debugfs", + Field::Id::Date, QueryFlags::IncludeRelated); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 4); + } + + { + auto qr = store.run_query("fix buffer overflow in debugfs and flag:unread", + Field::Id::Date, QueryFlags::None); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 1); + assert_equal(qr->begin().message_id().value_or(""), "20220805063702.GH3438@kadam"); + assert_equal(qr->begin().thread_id().value_or(""), "YuvYh1JbE3v+abd5@kili"); + } + + { + /* this one failed earlier, because the 'protonmail' id is the + * first reference, which means it does _not_ have the same + * thread-id as the rest; however, we filter these + * fake-message-ids now.*/ + g_test_bug("2312"); + + auto qr = store.run_query("fix buffer overflow in debugfs and flag:unread", + Field::Id::Date, QueryFlags::IncludeRelated); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 4); + } +} + + +static void +test_body_matricula() +{ + const TestMap test_msgs = {{ +{ +"basic/cur/matricula-msg:2,S", +R"(From: XXX <XX@XX.com> +Subject: + =?iso-8859-1?Q?EF_-_Pago_matr=EDcula_de_la_matr=EDcula_de_inscripci=F3n_a?= +Date: Thu, 4 Aug 2022 14:29:41 +0000 +Message-ID: + <VE1PR03MB5471882920DE08CFE44D97A0FE9F9@VE1PR03MB5471.eurprd03.prod.outlook.com> +Accept-Language: es-AR, es-ES, en-US +Content-Language: es-AR +X-MS-Has-Attach: yes +Content-Type: multipart/mixed; + boundary="_004_VE1PR03MB5471882920DE08CFE44D97A0FE9F9VE1PR03MB5471eurp_" +MIME-Version: 1.0 +X-OriginatorOrg: ef.com +X-MS-Exchange-CrossTenant-AuthAs: Internal +X-MS-Exchange-CrossTenant-AuthSource: VE1PR03MB5471.eurprd03.prod.outlook.com + +--_004_VE1PR03MB5471882920DE08CFE44D97A0FE9F9VE1PR03MB5471eurp_ +Content-Type: multipart/alternative; + boundary="_000_VE1PR03MB5471882920DE08CFE44D97A0FE9F9VE1PR03MB5471eurp_" + +--_000_VE1PR03MB5471882920DE08CFE44D97A0FE9F9VE1PR03MB5471eurp_ +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +Buenas tardes Familia, + + +Espero que est=E9n muy bien. + + + +Ya cargamos en sistema su pre inscripci=F3n para el curso + + +Quedamos atentos ante cualquier consulta que surja. + +Saludos, +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + + /* i.e., non-utf8 text parts were not converted */ + g_test_bug("2333"); + + // matches + for (auto&& expr: { + "subject:matrícula", + "subject:matricula", + "body:atentos", + "body:inscripción" + }) { + + if (g_test_verbose()) + g_message("query: '%s'", expr); + auto qr = store.run_query(expr); + assert_valid_result(qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + } +} + + + +static void +test_duplicate_refresh_real(bool rename) +{ + g_test_bug("2327"); + + const TestMap test_msgs = {{ + "inbox/new/msg", + { R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Wed, 26 Oct 2022 11:01:54 -0700 +To: example@example.com +Subject: Rainy night in Helsinki + +Boo! +)"}, + }}; + + /* create maildir with message */ + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + g_debug("%s", store.root_maildir().c_str()); + /* ensure we have a proper maildir, with new/, cur/ */ + auto mres = maildir_mkdir(store.root_maildir() + "/inbox"); + assert_valid_result(mres); + g_assert_cmpuint(store.size(), ==, 1U); + + /* + * find the one msg with a query + */ + auto qr = store.run_query("Helsinki", Field::Id::Date, QueryFlags::None); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 1); + const auto old_path = qr->begin().path().value(); + const auto old_docid = qr->begin().doc_id(); + assert_equal(qr->begin().message()->path(), old_path); + g_assert_true(::access(old_path.c_str(), F_OK) == 0); + + + /* + * mark as read, i.e. move to cur/; ensure it really moved. + */ + auto move_opts{rename ? Store::MoveOptions::ChangeName : Store::MoveOptions::None}; + auto moved_msgs = store.move_message(old_docid, Nothing, Flags::Seen, move_opts); + assert_valid_result(moved_msgs); + + g_assert_true(moved_msgs->size() == 1); + auto&& moved_msg_opt = store.find_message(moved_msgs->at(0).first); + g_assert_true(!!moved_msg_opt); + const auto&moved_msg = std::move(*moved_msg_opt); + const auto new_path = moved_msg.path(); + if (!rename) + assert_equal(new_path, store.root_maildir() + "/inbox/cur/msg:2,S"); + g_assert_cmpuint(store.size(), ==, 1); + g_assert_false(::access(old_path.c_str(), F_OK) == 0); + g_assert_true(::access(new_path.c_str(), F_OK) == 0); + + /* also ensure that the cached sexp for the message has been updated; + * that's what mu4e uses */ + const auto moved_sexp{moved_msg.sexp()}; + g_assert_true(moved_sexp.plistp()); + g_assert_true(!!moved_sexp.get_prop(":path")); + assert_equal(moved_sexp.get_prop(":path").value().string(), new_path); + + /* + * find new message with query, ensure it's really that new one. + */ + auto qr2 = store.run_query("Helsinki", Field::Id::Date, QueryFlags::None); + g_assert_true(!!qr2); + g_assert_cmpuint(qr2->size(), ==, 1); + assert_equal(qr2->begin().path().value(), new_path); + + /* index the messages */ + auto res = store.indexer().start({}); + g_assert_true(res); + while(store.indexer().is_running()) { + using namespace std::chrono_literals; + std::this_thread::sleep_for(100ms); + } + g_assert_cmpuint(store.size(), ==, 1); + + /* + * ensure query still has the right results + */ + auto qr3 = store.run_query("Helsinki", Field::Id::Date, QueryFlags::None); + g_assert_true(!!qr3); + g_assert_cmpuint(qr3->size(), ==, 1); + const auto path3{qr3->begin().path().value()}; + assert_equal(path3, new_path); + assert_equal(qr3->begin().message()->path(), new_path); + g_assert_true(::access(path3.c_str(), F_OK) == 0); +} + + +static void +test_duplicate_refresh() +{ + test_duplicate_refresh_real(false/*no rename*/); +} + + +static void +test_duplicate_refresh_rename() +{ + test_duplicate_refresh_real(true/*rename*/); +} + +static void +test_term_split() +{ + g_test_bug("2365"); + + // Note the fancy quote in "foo’s bar" + const TestMap test_msgs = {{ + "inbox/new/msg", + { +R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Wed, 26 Oct 2022 11:01:54 -0700 +To: example@example.com +Subject: foo’s bar + +Boo! +)"}, + }}; + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + /* true: match; false: no match */ + const auto cases = std::array<std::pair<const char*, bool>, 8>{{ + {"subject:foo's", true}, + {"subject:foo*", true}, + {"subject:/foo/", true}, + {"subject:/foo’s/", true}, /* <-- breaks before PR #2365 */ + {"subject:/foo.*bar/", true}, /* <-- breaks before PR #2365 */ + {"subject:/foo’s bar/", false}, /* <-- no matching, needs quoting */ + {"subject:\"/foo’s bar/\"", true}, /* <-- this works, quote the regex */ + {R"(subject:"/foo’s bar/")", true}, /* <-- this works, quote the regex */ + }}; + + for (auto&& test: cases) { + mu_debug("query: '{}'", test.first); + auto qr = store.run_query(test.first); + assert_valid_result(qr); + if (test.second) + g_assert_cmpuint(qr->size(), ==, 1); + else + g_assert_true(qr->empty()); + } +} + +static void +test_subject_kata_containers() +{ + g_test_bug("2167"); + + // Note the fancy quote in "foo’s bar" + const TestMap test_msgs = {{ + "inbox/new/msg", + { +R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Wed, 26 Oct 2022 11:01:54 -0700 +To: example@example.com +Subject: kata-containers + +voodoo-containers + +Boo! +)"}, + }}; + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + /* true: match; false: no match */ + const auto cases = std::vector<std::pair<const char*, bool>>{{ + {"subject:kata", true}, + {"subject:containers", true}, + {"subject:kata-containers", true}, + {"subject:\"kata containers\"", true}, + {"voodoo-containers", true}, + {"voodoo containers", true} + }}; + + for (auto&& test: cases) { + mu_debug("query: '{}'", test.first); + auto qr = store.run_query(test.first); + assert_valid_result(qr); + if (test.second) + g_assert_cmpuint(qr->size(), ==, 1); + else + g_assert_true(qr->empty()); + } +} + +static void +test_related_dup_threaded() +{ + // test message sent to self, and copy of received msg. + + const auto test_msg = R"(From: "Edward Mallory" <ed@leviathan.gb> +To: "Laurence Oliphant <oli@hotmail.com> +Subject: Boo +Date: Wed, 07 Dec 2022 18:38:06 +0200 +Message-ID: <875yentbhg.fsf@djcbsoftware.nl> +MIME-Version: 1.0 +Content-Type: text/plain + +Boo! +)"; + const TestMap test_msgs = { + {"sent/cur/msg1", test_msg }, + {"inbox/cur/msg1", test_msg }, + {"inbox/cur/msg2", test_msg }}; + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + + g_assert_cmpuint(store.size(), ==, 3); + + + // normal query should give 2 + { + auto qr = store.run_query("maildir:/inbox", Field::Id::Date, + QueryFlags::None); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 2); + } + + // a related query should give 3 + { + auto qr = store.run_query("maildir:/inbox", Field::Id::Date, + QueryFlags::IncludeRelated); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 3); + } + + // a related/threading query should give 3. + { + auto qr = store.run_query("maildir:/inbox", Field::Id::Date, + QueryFlags::IncludeRelated | QueryFlags::Threading); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 3); + } +} + + +static void +test_html() +{ + // test message sent to self, and copy of received msg. + + const auto test_msg = R"(From: Test <test@example.com> +To: abc@example.com +Date: Mon, 23 May 2011 10:53:45 +0200 +Subject: vla +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d" +Message-ID: <10374608.109906.11909.20115aabbccdd.MSGID@mailinglijst.nl> + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/plain; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +text + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/html; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +html + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d-- +)"; + const TestMap test_msgs = {{"inbox/cur/msg1", test_msg }}; + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + g_assert_cmpuint(store.size(), ==, 1); + + { + auto qr = store.run_query("body:text", Field::Id::Date, + QueryFlags::None); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 1); + } + + { + auto qr = store.run_query("body:html", Field::Id::Date, + QueryFlags::None); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 1); + } +} + + +static void +test_ngrams() +{ + g_test_bug("2167"); + + // Note the fancy quote in "foo’s bar" + const TestMap test_msgs = {{ + "inbox/new/msg", + { +R"(From: "Bob" <bob@builder.com> +Subject: スポンサーシップ募集 +To: "Chase" <chase@ppatrol.org> +Message-Id: 112342343e9dfo.fsf@builder.com + + 中文 + +https://trac.xapian.org/ticket/719 + + サーバがダウンしました +)"}}}; + + MemDb mdb; + Config conf{mdb}; + conf.set<Config::Id::SupportNgrams>(true); + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, conf)}; + + /* true: match; false: no match */ + const auto cases = std::vector<std::pair<std::string_view, bool>>{{ + {"body:中文", true}, + {"body:中", true}, + {"body:文", true}, + {"body:し", true}, + {"body:サー", true}, + {"body:サーバがダウンしました", true}, // fail + {"中文", true}, + {"中", true}, + {"文", true}, + {"subject:スポン", true }, + {"subject:スポンサーシップ募集", true }, + {"subject:シップ", true }, // XXX should match + {"サーバがダウンしました", true}, // okay + {"body:サーバがダウンしました", true}, // okay + {"subject:スポンサーシップ募集", true}, // okay + {"subject:シップx", true }, // XXX should match + }}; + + for (auto&& test: cases) { + auto qr = store.run_query(std::string{test.first}); + assert_valid_result(qr); + if (test.second) + g_assert_cmpuint(qr->size(), ==, 1); + else + g_assert_true(qr->empty()); + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/store/query/simple", + test_simple); + g_test_add_func("/store/query/spam-address-components", + test_spam_address_components); + g_test_add_func("/store/query/dups-related", + test_dups_related); + g_test_add_func("/store/query/related-missing-root", + test_related_missing_root); + g_test_add_func("/store/query/body-matricula", + test_body_matricula); + g_test_add_func("/store/query/duplicate-refresh", + test_duplicate_refresh); + g_test_add_func("/store/query/duplicate-refresh-rename", + test_duplicate_refresh_rename); + g_test_add_func("/store/query/term-split", + test_term_split); + g_test_add_func("/store/query/kata_containers", + test_subject_kata_containers); + g_test_add_func("/store/query/related-dup-threaded", + test_related_dup_threaded); + g_test_add_func("/store/query/html", + test_html); + g_test_add_func("/store/query/ngrams", + test_ngrams); + + return g_test_run(); +} diff --git a/lib/tests/test-mu-store.cc b/lib/tests/test-mu-store.cc new file mode 100644 index 0000000..da7f120 --- /dev/null +++ b/lib/tests/test-mu-store.cc @@ -0,0 +1,590 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <glib.h> +#include <stdlib.h> +#include <thread> +#include <array> +#include <unistd.h> +#include <time.h> +#include <fstream> + +#include <locale.h> + +#include "utils/mu-test-utils.hh" +#include "mu-store.hh" +#include "utils/mu-result.hh" +#include <utils/mu-utils.hh> +#include <utils/mu-utils-file.hh> +#include "mu-maildir.hh" + +using namespace Mu; + +using namespace std::chrono_literals; + +static std::string MuTestMaildir = Mu::canonicalize_filename(MU_TESTMAILDIR, "/"); +static std::string MuTestMaildir2 = Mu::canonicalize_filename(MU_TESTMAILDIR2, "/"); + +static void +test_store_ctor_dtor() +{ + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), "/tmp")}; + assert_valid_result(store); + + g_assert_true(store->empty()); + g_assert_cmpuint(0, ==, store->size()); + + g_assert_cmpuint(MU_STORE_SCHEMA_VERSION, ==, + store->config().get<Config::Id::SchemaVersion>()); +} + +static void +test_store_reinit() +{ + TempDir tempdir; + { + MemDb mdb; + Config conf{mdb}; + conf.set<Config::Id::MaxMessageSize>(1234567); + conf.set<Config::Id::BatchSize>(7654321); + conf.set<Config::Id::PersonalAddresses>( + StringVec{ "foo@example.com", "bar@example.com" }); + + auto store{Store::make_new(tempdir.path(), MuTestMaildir, conf)}; + assert_valid_result(store); + + g_assert_true(store->empty()); + g_assert_cmpuint(0, ==, store->size()); + + g_assert_cmpuint(MU_STORE_SCHEMA_VERSION, ==, + store->config().get<Config::Id::SchemaVersion>()); + + const auto msgpath{MuTestMaildir + "/cur/1283599333.1840_11.cthulhu!2,"}; + const auto id = store->add_message(msgpath); + assert_valid_result(id); + g_assert_true(store->contains_message(msgpath)); + g_assert_cmpuint(store->size(), ==, 1); + } + + //now let's reinitialize it. + { + auto store{Store::make(tempdir.path(), + Store::Options::Writable|Store::Options::ReInit)}; + + assert_valid_result(store); + g_assert_true(store->empty()); + + assert_equal(store->path(), tempdir.path()); + assert_equal(store->root_maildir(), MuTestMaildir); + + g_assert_cmpuint(store->config().get<Config::Id::BatchSize>(),==,7654321); + g_assert_cmpuint(store->config().get<Config::Id::MaxMessageSize>(),==,1234567); + + const auto addrs{store->config().get<Config::Id::PersonalAddresses>()}; + g_assert_cmpuint(addrs.size(),==,2); + g_assert_true(seq_some(addrs, [](auto&& a){return a=="foo@example.com";})); + g_assert_true(seq_some(addrs, [](auto&& a){return a=="bar@example.com";})); + + const auto msgpath{MuTestMaildir + "/cur/1283599333.1840_11.cthulhu!2,"}; + const auto id = store->add_message(msgpath); + assert_valid_result(id); + g_assert_true(store->contains_message(msgpath)); + g_assert_cmpuint(store->size(), ==, 1); + } +} + + + +static void +test_store_add_count_remove() +{ + TempDir tempdir{false}; + + auto store{Store::make_new(tempdir.path() + "/xapian", MuTestMaildir)}; + assert_valid_result(store); + + assert_equal(store->path(), tempdir.path() + "/xapian"); + assert_equal(store->root_maildir(), MuTestMaildir); + + const auto msgpath{MuTestMaildir + "/cur/1283599333.1840_11.cthulhu!2,"}; + const auto id1 = store->add_message(msgpath); + assert_valid_result(id1); + + g_assert_cmpuint(store->size(), ==, 1); + g_assert_true(store->contains_message(msgpath)); + + const auto id2 = store->add_message(MuTestMaildir2 + "/bar/cur/mail3"); + g_assert_false(!!id2); // wrong maildir. + + const auto msg3path{MuTestMaildir + "/cur/1252168370_3.14675.cthulhu!2,S"}; + const auto id3 = store->add_message(msg3path); + assert_valid_result(id3); + + g_assert_cmpuint(store->size(), ==, 2); + g_assert_true(store->contains_message(msg3path)); + + store->remove_message(id1.value()); + g_assert_cmpuint(store->size(), ==, 1); + g_assert_false( + store->contains_message(MuTestMaildir + "/cur/1283599333.1840_11.cthulhu!2,")); + + store->remove_message(msg3path); + g_assert_true(store->empty()); + g_assert_false(store->contains_message(msg3path)); +} + + +static void +test_message_mailing_list() +{ + constexpr const char *test_message_1 = +R"(Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 32F276963F + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:34 +0300 (EEST) +Message-Id: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: anon@example.com +To: sqlite-dev@sqlite.org +Mime-Version: 1.0 (Apple Message framework v926) +Date: Mon, 4 Aug 2008 11:40:49 +0200 +X-Mailer: Apple Mail (2.926) +Subject: Capybaras United +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Sender: sqlite-dev-bounces@sqlite.org +Content-Length: 639 + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; +)"; + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), "/home/test/Maildir")}; + assert_valid_result(store); + + const auto msgpath{"/home/test/Maildir/inbox/cur/1649279256.107710_1.evergrey:2,S"}; + auto message{Message::make_from_text(test_message_1, msgpath)}; + assert_valid_result(message); + + const auto docid = store->add_message(*message); + assert_valid_result(docid); + g_assert_cmpuint(store->size(),==, 1); + + /* ensure 'update' dtrt, i.e., nothing. */ + const auto docid2 = store->add_message(*message); + assert_valid_result(docid2); + g_assert_cmpuint(store->size(),==, 1); + g_assert_cmpuint(*docid,==,*docid2); + + auto msg2{store->find_message(*docid)}; + g_assert_true(!!msg2); + assert_equal(message->path(), msg2->path()); + + g_assert_true(store->contains_message(message->path())); + + const auto qr = store->run_query("to:sqlite-dev@sqlite.org"); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 1); +} + + +static void +test_message_attachments(void) +{ + constexpr const char* msg_text = +R"(Return-Path: <foo@example.com> +Received: from pop.gmail.com [256.85.129.309] + by evergrey with POP3 (fetchmail-6.4.29) + for <djcb@localhost> (single-drop); Thu, 24 Mar 2022 20:12:40 +0200 (EET) +Sender: "Foo, Example" <foo@example.com> +User-agent: mu4e 1.7.11; emacs 29.0.50 +From: "Foo Example" <foo@example.com> +To: bar@example.com +Subject: =?utf-8?B?w6R0dMOkY2htZcOxdHM=?= +Date: Thu, 24 Mar 2022 20:04:39 +0200 +Organization: ACME Inc. +Message-Id: <3144HPOJ0VC77.3H1XTAG2AMTLH@"@WILSONB.COM> +MIME-Version: 1.0 +X-label: @NextActions operation:mindcrime Queensrÿche +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain + +Hello, +--=-=-= +Content-Type: image/jpeg +Content-Disposition: attachment; filename=file-01.bin +Content-Transfer-Encoding: base64 + +AAECAw== +--=-=-= +Content-Type: audio/ogg +Content-Disposition: inline; filename=/tmp/file-02.bin +Content-Transfer-Encoding: base64 + +BAUGBw== +--=-=-= +Content-Type: message/rfc822 +Content-Disposition: attachment; + filename="message.eml" + +From: "Fnorb" <fnorb@example.com> +To: Bob <bob@example.com> +Subject: news for you +Date: Mon, 28 Mar 2022 22:53:26 +0300 + +Attached message! + +--=-=-= +Content-Type: text/plain + +World! +--=-=-=-- +)"; + + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), "/home/test/Maildir")}; + assert_valid_result(store); + + auto message{Message::make_from_text( + msg_text, + "/home/test/Maildir/inbox/cur/1649279256.abcde_1.evergrey:2,S")}; + assert_valid_result(message); + + const auto docid = store->add_message(*message); + assert_valid_result(docid); + + auto msg2{store->find_message(*docid)}; + g_assert_true(!!msg2); + assert_equal(message->path(), msg2->path()); + + g_assert_true(store->contains_message(message->path())); + + // for (auto&& term = msg2->document().xapian_document().termlist_begin(); + // term != msg2->document().xapian_document().termlist_end(); ++term) + // g_message(">>> %s", (*term).c_str()); + + const auto stats{store->statistics()}; + g_assert_cmpuint(stats.size,==,store->size()); + g_assert_cmpuint(stats.last_index,==,0); + g_assert_cmpuint(stats.last_change,>=,::time({})); +} + + +static void +test_index_move() +{ + const std::string msg_text = +R"(From: Valentine Michael Smith <mike@example.com> +To: Raul Endymion <raul@example.com> +Cc: emacs-devel@gnu.org +Subject: Re: multi-eq hash tables +Date: Tue, 03 May 2022 20:58:02 +0200 +Message-ID: <87h766tzzz.fsf@gnus.org> +MIME-Version: 1.0 +Content-Type: text/plain +Precedence: list +List-Id: "Emacs development discussions." <emacs-devel.gnu.org> +List-Post: <mailto:emacs-devel@gnu.org> + +Raul Endymion <raul@example.com> writes: + +> Maybe we should introduce something like: +> +> (define-hash-table-test shallow-equal +> (lambda (x1 x2) (while (and (consp x1) (consp x2) (eql (car x1) (car x2))) +> (setq x1 (cdr x1)) (setq x2 (cdr x2))) +> (equal x1 x2))) +> ...) + +Yes, that would be excellent. +)"; + + TempDir tempdir2; + + { // create a message file. + const auto res1 = maildir_mkdir(tempdir2.path() + "/Maildir/a"); + assert_valid_result(res1); + + std::ofstream output{tempdir2.path() + "/Maildir/a/new/msg"}; + output.write(msg_text.c_str(), msg_text.size()); + output.close(); + g_assert_true(output.good()); + } + + // Index it into a store. + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), tempdir2.path() + "/Maildir")}; + assert_valid_result(store); + + store->indexer().start({}); + size_t n{}; + while (store->indexer().is_running()) { + std::this_thread::sleep_for(100ms); + g_assert_cmpuint(n++,<=,25); + } + g_assert_true(!store->indexer().is_running()); + const auto& prog{store->indexer().progress()}; + g_assert_cmpuint(prog.updated,==,1); + g_assert_cmpuint(store->size(), ==, 1); + g_assert_false(store->empty()); + + // Find the message + auto qr = store->run_query("path:" + tempdir2.path() + "/Maildir/a/new/msg"); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(),==,1); + + const auto msg = qr->begin().message(); + g_assert_true(!!msg); + + // Check the message + const auto oldpath{msg->path()}; + assert_equal(msg->subject(), "Re: multi-eq hash tables"); + g_assert_true(msg->docid() != 0); + g_debug("%s", msg->sexp().to_string().c_str()); + + // Move the message from new->cur + std::this_thread::sleep_for(1s); /* ctime should change */ + const auto msgs3 = store->move_message(msg->docid(), {}, Flags::Seen); + assert_valid_result(msgs3); + g_assert_true(msgs3->size() == 1); + auto&& msg3_opt{store->find_message(msgs3->at(0).first/*id*/)}; + g_assert_true(!!msg3_opt); + auto&& msg3{std::move(*msg3_opt)}; + + assert_equal(msg3.maildir(), "/a"); + assert_equal(msg3.path(), tempdir2.path() + "/Maildir/a/cur/msg:2,S"); + g_assert_true(::access(msg3.path().c_str(), R_OK)==0); + g_assert_false(::access(oldpath.c_str(), R_OK)==0); + + g_debug("%s", msg3.sexp().to_string().c_str()); g_assert_cmpuint(store->size(), ==, 1); +} + + + +static void +test_store_move_dups() +{ + const std::string msg_text = +R"(From: Valentine Michael Smith <mike@example.com> +To: Raul Endymion <raul@example.com> +Subject: Re: multi-eq hash tables +Date: Tue, 03 May 2022 20:58:02 +0200 +Message-ID: <87h766tzzz.fsf@gnus.org> + +Yes, that would be excellent. +)"; + TempDir tempdir2; + + // create a message file + dups + const auto res1 = maildir_mkdir(tempdir2.path() + "/Maildir/a"); + assert_valid_result(res1); + const auto res2 = maildir_mkdir(tempdir2.path() + "/Maildir/b"); + assert_valid_result(res2); + + auto msg1_path = join_paths(tempdir2.path(), "Maildir/a/new/msg123"); + auto msg2_path = join_paths(tempdir2.path(), "Maildir/a/cur/msgabc:2,S"); + auto msg3_path = join_paths(tempdir2.path(),"Maildir/b/cur/msgdef:2,RS"); + + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), + join_paths(tempdir2.path() , "Maildir"))}; + assert_valid_result(store); + + std::vector<Store::Id> ids; + for (auto&& p: {msg1_path, msg2_path, msg3_path}) { + std::ofstream output{p}; + output.write(msg_text.c_str(), msg_text.size()); + output.close(); + auto res = store->add_message(p); + assert_valid_result(res); + ids.emplace_back(*res); + } + g_assert_cmpuint(store->size(), ==, 3); + + // mark main message (+ dups) as seen + auto mres = store->move_message(ids.at(0), {}, + Flags::Seen | Flags::Flagged | Flags::Passed, + Store::MoveOptions::DupFlags); + assert_valid_result(mres); + mu_info("found {} matches", mres->size()); + for (auto&& m: *mres) + mu_info("id: {}: {}", m.first, m.second); + + // al three dups should have been updated + g_assert_cmpuint(mres->size(), ==, 3); + auto&& id_msgs{store->find_messages(Store::id_vec(*mres))}; + + // first should be the original + g_assert_cmpuint(id_msgs.at(0).first, ==, ids.at(0)); + { // Message 1 + const Message& msg = id_msgs.at(0).second; + assert_equal(msg.path(), tempdir2.path() + "/Maildir/a/cur/msg123:2,FPS"); + g_assert_true(msg.flags() == (Flags::Seen|Flags::Flagged|Flags::Passed)); + } + // note: Seen and Passed should be added to msg2/3, but Flagged shouldn't + // msg3 should loose its R flag. + + auto check_msg2 = [&](const Message& msg) { + assert_equal(msg.path(), join_paths(tempdir2.path(), "/Maildir/a/cur/msgabc:2,PS")); + }; + auto check_msg3 = [&](const Message& msg) { + assert_equal(msg.path(), join_paths(tempdir2.path(), "/Maildir/b/cur/msgdef:2,PS")); + }; + + if (id_msgs.at(1).first == ids.at(1)) { + check_msg2(id_msgs.at(1).second); + check_msg3(id_msgs.at(2).second); + } else { + check_msg2(id_msgs.at(2).second); + check_msg3(id_msgs.at(1).second); + } +} + +static void +test_store_circular_symlink(void) +{ + allow_warnings(); + + g_test_bug("2517"); + + auto testhome{unwrap(make_temp_dir())}; + auto dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + + /* create a writable copy */ + const auto testmdir = join_paths(testhome, "test-maildir"); + auto cres1 = run_command({CP_PROGRAM, "-r", MU_TESTMAILDIR, testmdir}); + assert_valid_command(cres1); + // create a symink + auto cres2 = run_command({LN_PROGRAM, "-s", testmdir, join_paths(testmdir, "testlink")}); + assert_valid_command(cres2); + + auto&& store = unwrap(Store::make_new(dbpath, testmdir)); + store.indexer().start({}); + size_t n{}; + while (store.indexer().is_running()) { + std::this_thread::sleep_for(100ms); + g_assert_cmpuint(n++,<=,25); + } + // there will be a lot of dups.... + g_assert_false(store.empty()); + + remove_directory(testhome); +} + +static void +test_store_maildirs() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + + const auto mdirs = store->maildirs(); + + g_assert_cmpuint(mdirs.size(), ==, 3); + g_assert(seq_some(mdirs, [](auto&& m){return m == "/Foo";})); + g_assert(seq_some(mdirs, [](auto&& m){return m == "/bar";})); + g_assert(seq_some(mdirs, [](auto&& m){return m == "/wom_bat";})); +} + + +static void +test_store_parse() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + + // Xapian internal format (get_description()) is _not_ guaranteed + // to be the same between versions + const auto&& pq1{store->parse_query("subject:\"hello world\"", false)}; + const auto&& pq2{store->parse_query("subject:\"hello world\"", true)}; + + assert_equal(pq1, "(or (subject \"hello world\") (subject (phrase \"hello world\")))"); + + /* LCOV_EXCL_START*/ + if (pq2 != "Query((Shello world OR (Shello PHRASE 2 Sworld)))") { + g_test_skip("incompatible xapian descriptions"); + return; + } + /* LCOV_EXCL_STOP*/ + + assert_equal(pq2, "Query((Shello world OR (Shello PHRASE 2 Sworld)))"); +} + +static void +test_store_fail() +{ + { + const auto store = Store::make("/root/non-existent-path/12345"); + g_assert_false(!!store); + } + + { + const auto store = Store::make_new("/../../root/non-existent-path/12345", + "/../../root/non-existent-path/54321"); + g_assert_false(!!store); + } +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/store/ctor-dtor", test_store_ctor_dtor); + g_test_add_func("/store/reinit", test_store_reinit); + g_test_add_func("/store/add-count-remove", test_store_add_count_remove); + g_test_add_func("/store/message/mailing-list", + test_message_mailing_list); + g_test_add_func("/store/message/attachments", + test_message_attachments); + g_test_add_func("/store/move-dups", test_store_move_dups); + + g_test_add_func("/store/maildirs", test_store_maildirs); + g_test_add_func("/store/parse", test_store_parse); + + g_test_add_func("/store/index/index-move", test_index_move); + g_test_add_func("/store/index/circular-symlink", test_store_circular_symlink); + + g_test_add_func("/store/index/fail", test_store_fail); + + return g_test_run(); +} diff --git a/lib/tests/test-query.cc b/lib/tests/test-query.cc new file mode 100644 index 0000000..fd9ff1d --- /dev/null +++ b/lib/tests/test-query.cc @@ -0,0 +1,99 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <config.h> + +#include <vector> +#include <glib.h> + +#include <iostream> +#include <sstream> +#include <unistd.h> + +#include "mu-store.hh" +#include "mu-query.hh" +#include "utils/mu-result.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-test-utils.hh" + +using namespace Mu; + +static void +test_query() +{ + allow_warnings(); + TempDir temp_dir; + + auto store = Store::make_new(temp_dir.path(), std::string{MU_TESTMAILDIR}); + assert_valid_result(store); + + auto&& idx{store->indexer()}; + g_assert_true(idx.start(Indexer::Config{})); + while (idx.is_running()) { + g_usleep(1000); + } + + auto dump_matches = [](const QueryResults& res) { + size_t n{}; + for (auto&& item : res) { + if (g_test_verbose()) { + std::cout << item.query_match() << '\n'; + mu_debug("{:02d} {} {}", + ++n, + item.path().value_or("<none>"), + item.message_id().value_or("<none>")); + } + } + }; + + g_assert_cmpuint(store->size(), ==, 19); + + { + const auto res = store->run_query("", {}, QueryFlags::None); + g_assert_true(!!res); + g_assert_cmpuint(res->size(), ==, 19); + dump_matches(*res); + + g_assert_cmpuint(store->count_query(""), ==, 19); + + } + + { + const auto res = store->run_query("", Field::Id::Path, QueryFlags::None, 11); + g_assert_true(!!res); + g_assert_cmpuint(res->size(), ==, 11); + dump_matches(*res); + } +} + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/query", test_query); + + return g_test_run(); + +} catch (const std::runtime_error& re) { + std::cerr << re.what() << "\n"; + return 1; +} catch (...) { + std::cerr << "caught exception\n"; + return 1; +} diff --git a/lib/utils/meson.build b/lib/utils/meson.build new file mode 100644 index 0000000..3263a94 --- /dev/null +++ b/lib/utils/meson.build @@ -0,0 +1,66 @@ +## Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +thirdparty=join_paths('..', '..', 'thirdparty') + +srcs = [ + 'mu-command-handler.cc', + 'mu-html-to-text.cc', + 'mu-lang-detector.cc', + 'mu-logger.cc', + 'mu-option.cc', + 'mu-readline.cc', + 'mu-sexp.cc', + 'mu-utils-file.cc', + 'mu-utils.cc', +] + +if not get_option('tests').disabled() + test_srcs = [ 'mu-test-utils.cc' ] +else + test_srcs = [] +endif + +lib_mu_utils=static_library('mu-utils', + [ srcs, test_srcs ], dependencies: [ + glib_dep, + gio_dep, + gio_unix_dep, + config_h_dep, + readline_dep, + cld2_dep +], include_directories: + include_directories(['.', '..', thirdparty]), +install: false) + +lib_mu_utils_dep = declare_dependency( + link_with: lib_mu_utils, + compile_args: '-DFMT_HEADER_ONLY', + include_directories: + include_directories(['.', '..', thirdparty])) + +# +# tools +# +html2text = executable('mu-html2text', + 'mu-html-to-text.cc', + dependencies: [ lib_mu_utils_dep, glib_dep ], + cpp_args: ['-DBUILD_HTML_TO_TEXT'], + install: false) + +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/lib/utils/mu-async-queue.hh b/lib/utils/mu-async-queue.hh new file mode 100644 index 0000000..afabef5 --- /dev/null +++ b/lib/utils/mu-async-queue.hh @@ -0,0 +1,184 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_ASYNC_QUEUE_HH__ +#define __MU_ASYNC_QUEUE_HH__ + +#include <deque> +#include <mutex> +#include <chrono> +#include <condition_variable> + +namespace Mu { + +constexpr std::size_t UnlimitedAsyncQueueSize{0}; + +template <typename ItemType, /**< the type of Item to queue */ + std::size_t MaxSize = UnlimitedAsyncQueueSize, /**< maximum size for the queue */ + typename Allocator = std::allocator<ItemType>> /**< allocator for the items */ + +class AsyncQueue { + public: + using value_type = ItemType; + using allocator_type = Allocator; + using size_type = std::size_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename std::allocator_traits<allocator_type>::pointer; + using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer; + + using Timeout = std::chrono::steady_clock::duration; + + /** + * Push an item to the end of the queue by moving it + * + * @param item the item to move to the end of the queue + * @param timeout and optional timeout + * + * @return true if the item was pushed; false otherwise. + */ + bool push(const value_type& item, Timeout timeout = {}) { + return push(std::move(value_type(item)), timeout); + } + + /** + * Push an item to the end of the queue by moving it + * + * @param item the item to move to the end of the queue + * @param timeout and optional timeout + * + * @return true if the item was pushed; false otherwise. + */ + bool push(value_type&& item, Timeout timeout = {}) { + std::unique_lock lock{m_}; + + if (!unlimited()) { + const auto rv = cv_full_.wait_for(lock, timeout, [&]() { + return !full_unlocked(); + }) && !full_unlocked(); + if (!rv) + return false; + } + + q_.emplace_back(std::move(item)); + cv_empty_.notify_one(); + + return true; + } + + /** + * Pop an item from the queue + * + * @param receives the value if the function returns true + * @param timeout optional time to wait for an item to become available + * + * @return true if an item was popped (into val), false otherwise. + */ + bool pop(value_type& val, Timeout timeout = {}) { + std::unique_lock lock{m_}; + + if (timeout != Timeout{}) { + const auto rv = cv_empty_.wait_for(lock, timeout, [&]() { + return !q_.empty(); + }) && !q_.empty(); + if (!rv) + return false; + + } else if (q_.empty()) + return false; + + val = std::move(q_.front()); + q_.pop_front(); + cv_full_.notify_one(); + + return true; + } + + /** + * Clear the queue + * + */ + void clear() { + std::unique_lock lock{m_}; + q_.clear(); + cv_full_.notify_one(); + } + + /** + * Size of the queue + * + * + * @return the size + */ + size_type size() const { + std::unique_lock lock{m_}; + return q_.size(); + } + + /** + * Maximum size of the queue if specified through the template + * parameter; otherwise the (theoretical) max_size of the inner + * container. + * + * @return the maximum size + */ + size_type max_size() const { return unlimited() ? q_.max_size() : MaxSize; } + + /** + * Is the queue empty? + * + * @return true or false + */ + bool empty() const { + std::unique_lock lock{m_}; + return q_.empty(); + } + + /** + * Is the queue full? Returns false unless a maximum size was specified + * (as a template argument) + * + * @return true or false. + */ + bool full() const { + if (unlimited()) + return false; + + std::unique_lock lock{m_}; + return full_unlocked(); + } + + /** + * Is this queue (theoretically) unlimited in size? + * + * @return true or false + */ + constexpr static bool unlimited() { return MaxSize == UnlimitedAsyncQueueSize; } + +private: + bool full_unlocked() const { return q_.size() >= max_size(); } + + std::deque<ItemType, Allocator> q_; + mutable std::mutex m_; + std::condition_variable cv_full_, cv_empty_; +}; + +} // namespace Mu + +#endif /* __MU_ASYNC_QUEUE_HH__ */ diff --git a/lib/utils/mu-command-handler.cc b/lib/utils/mu-command-handler.cc new file mode 100644 index 0000000..927df0b --- /dev/null +++ b/lib/utils/mu-command-handler.cc @@ -0,0 +1,273 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-command-handler.hh" +#include "mu-error.hh" +#include "mu-utils.hh" + +#include <iostream> +#include <algorithm> + +using namespace Mu; + +Option<std::vector<std::string>> +Command::string_vec_arg(const std::string& name) const +{ + auto&& val{arg_val(name, Sexp::Type::List)}; + if (!val) + return Nothing; + + std::vector<std::string> vec; + for (const auto& item : val->list()) { + if (!item.stringp()) { + // mu_warning("command: non-string in string-list for {}: {}", + // name, to_string()); + return Nothing; + } else + vec.emplace_back(item.string()); + } + + return vec; +} + +static Result<void> +validate(const CommandHandler::CommandInfoMap& cmap, + const CommandHandler::CommandInfo& cmd_info, + const Command& cmd) +{ + // all required parameters must be present + for (auto&& arg : cmd_info.args) { + + const auto& argname{arg.first}; + const auto& arginfo{arg.second}; + + // calls use keyword-parameters, e.g. + // + // (my-function :bar 1 :cuux "fnorb") + // + // so, we're looking for the odd-numbered parameters. + const auto param_it = cmd.find_arg(argname); + const auto&& param_val = std::next(param_it); + // it's an error when a required parameter is missing. + if (param_it == cmd.cend()) { + if (arginfo.required) + return Err(Error::Code::Command, + "missing required parameter {} in command '{}'", + argname, cmd.to_string()); + continue; // not required + } + + // the types must match, but the 'nil' symbol is acceptable as "no value" + if (param_val->type() != arginfo.type && !(param_val->nilp())) + return Err(Error::Code::Command, + "parameter {} expects type {}, but got {} in command '{}'", + argname, to_string(arginfo.type), + to_string(param_val->type()), cmd.to_string()); + } + + // all parameters must be known + for (auto it = cmd.cbegin() + 1; it != cmd.cend() && it + 1 != cmd.cend(); it += 2) { + const auto& cmdargname{it->symbol()}; + if (std::none_of(cmd_info.args.cbegin(), cmd_info.args.cend(), + [&](auto&& arg) { return cmdargname == arg.first; })) + return Err(Error::Code::Command, + "unknown parameter '{} 'in command '{}'", + cmdargname.name.c_str(), cmd.to_string().c_str()); + } + + return Ok(); + +} + +Result<void> +CommandHandler::invoke(const Command& cmd, bool do_validate) const +{ + const auto cmit{cmap_.find(cmd.name())}; + if (cmit == cmap_.cend()) + return Err(Error::Code::Command, + "unknown command '{}'", cmd.to_string().c_str()); + + const auto& cmd_info{cmit->second}; + if (do_validate) { + if (auto&& res = validate(cmap_, cmd_info, cmd); !res) + return Err(res.error()); + } + + if (cmd_info.handler) + cmd_info.handler(cmd); + + return Ok(); +} + + +// LCOV_EXCL_START +#ifdef BUILD_TESTS + +#include "mu-test-utils.hh" + + +static void +test_args() +{ + const auto cmd = Command::make_parse(R"((foo :bar 123 :cuux "456" :boo nil :bah true))"); + assert_valid_result(cmd); + + assert_equal(cmd->name(), "foo"); + g_assert_true(cmd->find_arg(":bar") != cmd->cend()); + g_assert_true(cmd->find_arg(":bxr") == cmd->cend()); + + g_assert_cmpint(cmd->number_arg(":bar").value_or(-1), ==, 123); + g_assert_cmpint(cmd->number_arg(":bor").value_or(-1), ==, -1); + + assert_equal(cmd->string_arg(":cuux").value_or(""), "456"); + assert_equal(cmd->string_arg(":caax").value_or(""), ""); // not present + assert_equal(cmd->string_arg(":bar").value_or("abc"), "abc"); // wrong type + + g_assert_false(cmd->boolean_arg(":boo")); + g_assert_true(cmd->boolean_arg(":bah")); +} + +using CommandInfoMap = CommandHandler::CommandInfoMap; +using ArgMap = CommandHandler::ArgMap; +using ArgInfo = CommandHandler::ArgInfo; +using CommandInfo = CommandHandler::CommandInfo; + +static Result<void> +call(const CommandInfoMap& cmap, const std::string& str) try { + + if (const auto cmd{Command::make_parse(str)}; !cmd) + return Err(Error::Code::Internal, "invalid s-expression '{}'", str); + else + return CommandHandler(cmap).invoke(*cmd); + +} catch (const Error& err) { + return Err(Error{err}); +} + +static void +test_command() +{ + allow_warnings(); + + CommandInfoMap ci_map; + ci_map.emplace( + "my-command", + CommandInfo{ArgMap{{":param1", ArgInfo{Sexp::Type::String, true, "some string"}}, + {":param2", ArgInfo{Sexp::Type::Number, false, "some integer"}}}, + "My command,", + {}}); + ci_map.emplace( + "another-command", + CommandInfo{ + ArgMap{ + {":queries", ArgInfo{Sexp::Type::List, false, + "queries for which to get read/unread numbers"}}, + {":symbol", ArgInfo{Sexp::Type::Symbol, true, + "some boring symbol"}}, + {":bool", ArgInfo{Sexp::Type::Symbol, true, + "some even more boring boolean symbol"}}, + {":symbol2", ArgInfo{Sexp::Type::Symbol, false, + "some even more boring symbol"}}, + {":bool2", ArgInfo{Sexp::Type::Symbol, false, + "some boring boolean symbol"}}, + }, + "get unread/totals information for a list of queries", + [&](const auto& params) { + const auto queries{params.string_vec_arg(":queries") + .value_or(std::vector<std::string>{})}; + g_assert_cmpuint(queries.size(),==,3); + g_assert_true(params.bool_arg(":bool").value_or(false) == true); + assert_equal(params.symbol_arg(":symbol").value_or("boo"), "sym"); + + g_assert_false(!!params.bool_arg(":bool2")); + g_assert_false(!!params.bool_arg(":symbol2")); + + }}); + + CommandHandler handler(std::move(ci_map)); + const auto cmap{handler.info_map()}; + + assert_valid_result(call(cmap, "(my-command :param1 \"hello\")")); + assert_valid_result(call(cmap, "(my-command :param1 \"hello\" :param2 123)")); + g_assert_false(!!call(cmap, "(my-command :param1 \"hello\" :param2 123 :param3 xxx)")); + assert_valid_result(call(cmap, "(another-command :queries (\"foo\" \"bar\" \"cuux\") " + ":symbol sym :bool true)")); +} + +static void +test_command2() +{ + allow_warnings(); + + CommandInfoMap cmap; + cmap.emplace("bla", + CommandInfo{ArgMap{ + {":foo", ArgInfo{Sexp::Type::Number, false, "foo"}}, + {":bar", ArgInfo{Sexp::Type::String, false, "bar"}}, + }, "yeah", + [&](const auto& params) {}}); + + g_assert_true(call(cmap, "(bla :foo nil)")); + g_assert_false(call(cmap, "(bla :foo nil :bla nil)")); +} + +static void +test_command_fail() +{ + allow_warnings(); + + CommandInfoMap cmap; + + cmap.emplace( + "my-command", + CommandInfo{ArgMap{{":param1", ArgInfo{Sexp::Type::String, true, "some string"}}, + {":param2", ArgInfo{Sexp::Type::Number, false, "some integer"}}}, + "My command,", + {}}); + + g_assert_false(call(cmap, "(my-command)")); + g_assert_false(call(cmap, "(my-command2)")); + g_assert_false(call(cmap, "(my-command :param1 123 :param2 123)")); + g_assert_false(call(cmap, "(my-command :param1 \"hello\" :param2 \"123\")")); + + g_assert_false(call(cmap, "(my-command")); + + g_assert_false(!!Command::make_parse(R"((foo :bar 123 :cuux "456" :boo nil :bah))")); +} + + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/utils/command-parser/args", test_args); + g_test_add_func("/utils/command-parser/command", test_command); + g_test_add_func("/utils/command-parser/command2", test_command2); + g_test_add_func("/utils/command-parser/command-fail", test_command_fail); + + return g_test_run(); + +} catch (const std::runtime_error& re) { + std::cerr << re.what() << "\n"; + return 1; +} + +#endif /*BUILD_TESTS*/ +// LCOV_EXCL_STOP diff --git a/lib/utils/mu-command-handler.hh b/lib/utils/mu-command-handler.hh new file mode 100644 index 0000000..755af53 --- /dev/null +++ b/lib/utils/mu-command-handler.hh @@ -0,0 +1,298 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#ifndef MU_COMMAND_HANDLER_HH__ +#define MU_COMMAND_HANDLER_HH__ + +#include <vector> +#include <string> +#include <ostream> +#include <stdexcept> +#include <unordered_map> +#include <functional> +#include <algorithm> + +#include "utils/mu-error.hh" +#include "utils/mu-sexp.hh" +#include "utils/mu-option.hh" + +namespace Mu { + +/// +/// Commands are s-expressions with the follow properties: + +/// 1) a command is a list with a command-name as its first argument +/// 2) the rest of the parameters are pairs of colon-prefixed symbol and a value of some +/// type (ie. 'keyword arguments') +/// 3) each command is described by its CommandInfo structure, which defines the type +/// 4) calls to the command must include all required parameters +/// 5) all parameters must be of the specified type; however the symbol 'nil' is allowed +/// for specify a non-required parameter to be absent; this is for convenience on the +/// call side. + +struct Command: public Sexp { + + static Result<Command> make(Sexp&& sexp) try { + return Ok(Command{std::move(sexp)}); + } catch (const Error& e) { + return Err(e); + } + + static Result<Command> make_parse(const std::string& cmdstr) try { + if (auto&& sexp{Sexp::parse(cmdstr)}; !sexp) + return Err(sexp.error()); + else + return Ok(Command(std::move(*sexp))); + } catch (const Error& e) { + return Err(e); + } + + /** + * Get name of the command (first element) in a command exp + * + * @return name + */ + const std::string& name() const { + return cbegin()->symbol().name; + } + + /** + * Find the argument with the given name. + * + * @param arg name + * + * @return iterator point at the argument, or cend + */ + const_iterator find_arg(const std::string& arg) const { + return find_prop(arg, cbegin() + 1, cend()); + } + + /** + * Get a string argument + * + * @param name of the argument + * + * @return ref to string, or Nothing if not found + */ + Option<const std::string&> string_arg(const std::string& name) const { + if (auto&& val{arg_val(name, Sexp::Type::String)}; !val) + return Nothing; + else + return val->string(); + } + + /** + * Get a string-vec argument + * + * @param name of the argument + * + * @return ref to string-vec, or Nothing if not found or some error. + */ + Option<std::vector<std::string>> string_vec_arg(const std::string& name) const; + + /** + * Get a symbol argument + * + * @param name of the argument + * + * @return ref to symbol name, or Nothing if not found + */ + Option<const std::string&> symbol_arg(const std::string& name) const { + if (auto&& val{arg_val(name, Sexp::Type::Symbol)}; !val) + return Nothing; + else + return val->symbol().name; + } + + /** + * Get a number argument + * + * @param name of the argument + * + * @return number or Nothing if not found + */ + Option<int> number_arg(const std::string& name) const { + if (auto&& val{arg_val(name, Sexp::Type::Number)}; !val) + return Nothing; + else + return static_cast<int>(val->number()); + } + + /* + * helpers + */ + + /** + * Get a boolean argument + * + * @param name of the argument + * + * @return true if there's a non-nil symbol value for the given + * name; false otherwise. + */ + Option<bool> bool_arg(const std::string& name) const { + if (auto&& symb{symbol_arg(name)}; !symb) + return Nothing; + else + return symb.value() == "nil" ? false : true; + } + + /** + * Treat any argument as a boolean + * + * @param name name of the argument + * + * @return false if the the argument is absent or the symbol false; + * otherwise true. + */ + bool boolean_arg(const std::string& name) const { + auto&& it{find_arg(name)}; + return (it == cend() || std::next(it)->nilp()) ? false : true; + } + +private: + explicit Command(Sexp&& s){ + *this = std::move(static_cast<Command&&>(s)); + if (!listp() || empty() || !cbegin()->symbolp() || + !plistp(cbegin() + 1, cend())) + throw Error(Error::Code::Command, + "expected command, got '{}'", to_string()); + } + + + Option<const Sexp&> arg_val(const std::string& name, Sexp::Type type) const { + if (auto&& it{find_arg(name)}; it == cend()) { + //std::cerr << "--> %s name found " << name << '\n'; + return Nothing; + } else if (auto&& val{it + 1}; val->type() != type) { + //std::cerr << "--> type " << Sexp::type_name(it->type()) << '\n'; + return Nothing; + } else + return *val; + } +}; + +struct CommandHandler { + + /// Information about a function argument + struct ArgInfo { + ArgInfo(Sexp::Type typearg, bool requiredarg, std::string&& docarg) + : type{typearg}, required{requiredarg}, docstring{std::move(docarg)} {} + const Sexp::Type type; /**< Sexp::Type of the argument */ + const bool required; /**< Is this argument required? */ + const std::string docstring; /**< Documentation */ + }; + + /// The arguments for a function, which maps their names to the information. + using ArgMap = std::unordered_map<std::string, ArgInfo>; + + // A handler function + using Handler = std::function<void(const Command&)>; + + /// Information about some command + struct CommandInfo { + CommandInfo(ArgMap&& argmaparg, std::string&& docarg, Handler&& handlerarg) + : args{std::move(argmaparg)}, docstring{std::move(docarg)}, + handler{std::move(handlerarg)} {} + const ArgMap args; + const std::string docstring; + const Handler handler; + + /** + * Get a sorted list of argument names, for display. Required args come + * first, then alphabetical. + * + * @return vec with the sorted names. + */ /* LCOV_EXCL_START */ + std::vector<std::string> sorted_argnames() const { + // sort args -- by required, then alphabetical. + std::vector<std::string> names; + for (auto&& arg : args) + names.emplace_back(arg.first); + std::sort(names.begin(), names.end(), [&](const auto& name1, const auto& name2) { + const auto& arg1{args.find(name1)->second}; + const auto& arg2{args.find(name2)->second}; + if (arg1.required != arg2.required) + return arg1.required; + else + return name1 < name2; + }); + return names; + } + /* LCOV_EXCL_STOP */ + + }; + + /// All commands, mapping their name to information about them. + using CommandInfoMap = std::unordered_map<std::string, CommandInfo>; + + CommandHandler(const CommandInfoMap& cmap): cmap_{cmap} {} + CommandHandler(CommandInfoMap&& cmap): cmap_{std::move(cmap)} {} + + const CommandInfoMap& info_map() const { return cmap_; } + + /** + * Invoke some command + * + * A command uses keyword arguments, e.g. something like: (foo :bar 1 + * :cuux "fnorb") + * + * @param cmd a Sexp describing a command call + * @param validate whether to validate before invoking. Useful during + * development. + * + * Return Ok() or some Error + */ + Result<void> invoke(const Command& cmd, bool validate=true) const; + +private: + const CommandInfoMap cmap_; +}; + +/* LCOV_EXCL_START */ +static inline std::ostream& +operator<<(std::ostream& os, const CommandHandler::ArgInfo& info) +{ + os << info.type << " (" << (info.required ? "required" : "optional") << ")"; + + return os; +} +/* LCOV_EXCL_STOP */ + +static inline std::ostream& +operator<<(std::ostream& os, const CommandHandler::CommandInfo& info) +{ + for (auto&& arg : info.args) + os << " " << arg.first << " " << arg.second << '\n' + << " " << arg.second.docstring << "\n"; + + return os; +} + +static inline std::ostream& +operator<<(std::ostream& os, const CommandHandler::CommandInfoMap& map) +{ + for (auto&& c : map) + os << c.first << '\n' << c.second; + + return os; +} + +} // namespace Mu + +#endif /* MU_COMMAND_HANDLER_HH__ */ diff --git a/lib/utils/mu-error.cc b/lib/utils/mu-error.cc new file mode 100644 index 0000000..1d098fc --- /dev/null +++ b/lib/utils/mu-error.cc @@ -0,0 +1,64 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#if BUILD_TESTS + +#include "mu-error.hh" +#include "mu-test-utils.hh" + +using namespace Mu; + +static void +test_fill_error() +{ + const Error err{Error::Code::Internal, "boo!"}; + GError *gerr{}; + + err.fill_g_error(&gerr); + + assert_equal(gerr->message, "boo!"); + g_assert_cmpint(gerr->code, ==, static_cast<int>(err.code())); + + g_clear_error(&gerr); +} + +static void +test_add_hint() +{ + Error err(Error::Code::Internal, "baa!"); + err.add_hint("hello"); + + assert_equal(err.hint(), "hello"); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/error/fill-error", test_fill_error); + g_test_add_func("/error/add-hint", test_add_hint); + + return g_test_run(); + +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-error.hh b/lib/utils/mu-error.hh new file mode 100644 index 0000000..36b4178 --- /dev/null +++ b/lib/utils/mu-error.hh @@ -0,0 +1,200 @@ +/* +** Copyright (C) 2019-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_ERROR_HH__ +#define MU_ERROR_HH__ + +#include <stdexcept> +#include <string> +#include <errno.h> +#include <cstdint> + +#include "mu-utils.hh" +#include <glib.h> + +#ifndef FMT_HEADER_ONLY +#define FMT_HEADER_ONLY +#endif + +#include <fmt/format.h> +#include <fmt/core.h> + +namespace Mu { + +// calculate an error enum value. +constexpr uint32_t err_enum(uint8_t code, uint8_t rv, uint8_t cat) { + return static_cast<uint32_t>(code|(rv << 16)|cat<<24); +} + +struct Error final : public std::exception { + + // 16 lower bits are for the error code;the next 8 bits are for the return code; the upper + // byte is for flags + static constexpr uint8_t SoftError = 1; + + enum struct Code: uint32_t { + Ok = err_enum(0,0,0), + + // used by mu4e. + NoMatches = err_enum(4,2,SoftError), + SchemaMismatch = err_enum(110,11,0), + + // other + AccessDenied = err_enum(100,1,0), + AssertionFailure = err_enum(101,1,0), + Command = err_enum(102,1,0), + Crypto = err_enum(103,1,0), + File = err_enum(104,1,0), + Index = err_enum(105,1,0), + Internal = err_enum(106,1,0), + InvalidArgument = err_enum(107,1,0), + Message = err_enum(108,1,0), + NotFound = err_enum(109,1,0), + Parsing = err_enum(111,1,0), + Play = err_enum(112,1,0), + Query = err_enum(113,1,0), + Script = err_enum(115,1,0), + ScriptNotFound = err_enum(116,1,0), + Store = err_enum(117,1,0), + StoreLock = err_enum(118,19,0), + UnverifiedSignature = err_enum(119,1,0), + User = err_enum(120,1,0), + Xapian = err_enum(121,1,0), + + CannotReinit = err_enum(122,1,0), + }; + + /** + * Construct an error + * + * @param code the error-code + * @param args... libfmt-style format string and parameters + */ + template<typename...T> + Error(Code code, fmt::format_string<T...> frm, T&&... args): + code_{code}, + what_{fmt::format(frm, std::forward<T>(args)...)} {} + + /** + * Construct an error + * + * @param code the error-code + * @param gerr a GError (or {}); the error is _consumed_ by this function + * @param args... libfmt-style format string and parameters + */ + template<typename...T> + Error(Code code, GError **gerr, fmt::format_string<T...> frm, T&&... args): + code_{code}, + what_{fmt::format(frm, std::forward<T>(args)...) + + fmt::format(": {}", (gerr && *gerr) ? (*gerr)->message : + "something went wrong")} + { g_clear_error(gerr); } + + /** + * Get the descriptive message for this error. + * + * @return + */ + virtual const char* what() const noexcept override { return what_.c_str(); } + + /** + * Get the error-code for this error + * + * @return the error-code + */ + Code code() const noexcept { return code_; } + + /** + * Get the error number (e.g. for reporting to mu4e) for some error. + * + * @param c error code + * + * @return the error number + */ + static constexpr uint32_t error_number(Code c) noexcept { + return static_cast<uint32_t>(c) & 0xffff; + } + + /** + * Is this is a 'soft error'? + * + * @return true or false + */ + constexpr bool is_soft_error() const { + return !!((static_cast<uint32_t>(code_)>>24) & SoftError); + } + + constexpr uint8_t exit_code() const { + return ((static_cast<uint32_t>(code_) >> 16) & 0xff); + } + + /** + * Fill a GError with the error information + * + * @param err GError** (or NULL) + */ + void fill_g_error(GError **err) const noexcept{ + g_set_error(err, error_quark(), static_cast<int>(code_), + "%s", what_.c_str()); + } + + /** + * Add an end-user hint + * + * @param args... libfmt-style format string and parameters + * + * @return the error + */ + template<typename...T> + Error& add_hint(fmt::format_string<T...> frm, T&&... args) { + hint_ = fmt::format(frm, std::forward<T>(args)...); + return *this; + } + + /** + * Get the hint + * + * @return the hint, empty for no hint. + */ + const std::string& hint() const { return hint_; } + +private: + static inline GQuark error_quark (void) { + static GQuark error_domain = 0; + if (G_UNLIKELY(error_domain == 0)) + error_domain = g_quark_from_static_string("mu-error-quark"); + return error_domain; + } + + const Code code_; + const std::string what_; + std::string hint_; +}; + +static inline auto +format_as(const Error& err) { + return mu_format("<{} ({}:{})>", + err.what(), + Error::error_number(err.code()), + err.exit_code()); +} + +} // namespace Mu + +#endif /* MU_ERROR_HH__ */ diff --git a/lib/utils/mu-html-to-text.cc b/lib/utils/mu-html-to-text.cc new file mode 100644 index 0000000..08f1f4d --- /dev/null +++ b/lib/utils/mu-html-to-text.cc @@ -0,0 +1,598 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-utils.hh" +#include "mu-option.hh" +#include "mu-regex.hh" + +#include <string> +#include <array> +#include <string_view> +#include <algorithm> + +using namespace Mu; + + +static bool +starts_with(std::string_view haystack, std::string_view needle) +{ + if (needle.size() > haystack.size()) + return false; + + for (auto&& c = 0U; c != needle.size(); ++c) + if (::tolower(haystack[c]) != ::tolower(needle[c])) + return false; + + return true; +} + +static bool +matches(std::string_view haystack, std::string_view needle) +{ + if (needle.size() != haystack.size()) + return false; + else + return starts_with(haystack, needle); +} + + + +/** + * HTML parsing context + * + */ +class Context { +public: + /** + * Construct a parsing context + * + * @param html some html to parse + */ + Context(const std::string& html): html_{html}, pos_{} {} + + /** + * Are we done with the html blob, i.e, has it been fully scraped? + * + * @return true or false + */ + bool done() const { + return pos_ >= html_.size(); + } + + /** + * Get the current position + * + * @return position + */ + size_t position() const { + return pos_; + } + + /** + * Get the size of the HTML + * + * @return size + */ + size_t size() const { + return html_.size(); + } + + /** + * Advance the position by _n_ characters. + * + * @param n number by which to advance. + */ + void advance(size_t n=1) { + if (pos_ + n > html_.size()) + throw std::range_error("out of range"); + pos_ += n; + } + + /** + * Are we looking at the given string? + * + * @param str string to match (case-insensitive) + * + * @return true or false + */ + bool looking_at(std::string_view str) const { + if (pos_ >= html_.size() || pos_ + str.size() >= html_.size()) + return false; + else + return matches({html_.data()+pos_, str.size()}, str); + } + + /** + * Grab a substring-view from the html + * + * @param fpos starting position + * @param len length + * + * @return string view + */ + std::string_view substr(size_t fpos, size_t len) const { + if (fpos + len > html_.size()) + throw std::range_error(mu_format("{} + {} > {}", + fpos, len, html_.size())); + else + return { html_.data() + fpos, len }; + } + + /** + * Grab the string of alphabetic characters at the + * head (pos) of the context, and advance over it. + * + * @return the head-word or empty + */ + std::string_view eat_head_word() { + size_t start_pos{pos_}; + while (!done()) { + if (!::isalpha(html_.at(pos_))) + break; + ++pos_; + } + return {html_.data() + start_pos, pos_ - start_pos}; + } + + + /** + * Get the scraped data; only available when done() + + * @return scraped data + */ + std::string scraped() { + return cleanup(raw_scraped_); + } + + /** + * Get the raw scrape buffer, where we can append + * scraped data. + * + * @return the buffer + */ + std::string& raw_scraped() { + return raw_scraped_; + } + + + /** + * Get a reference to the HTML + * + * @return html + */ + const std::string& html() const { return html_; } + +private: + + /** + * Cleanup some raw scraped html: remove superfluous + * whitespace, avoid too long lines. + * + * @param unclean + * + * @return cleaned up string. + */ + std::string cleanup(const std::string unclean) const { + // reduce whitespace and avoid too long lines; + // makes it easier to debug. + bool was_wspace{}; + size_t col{}; + std::string clean; + clean.reserve(unclean.size()/2); + for(auto&& c: unclean) { + auto wspace = c == ' ' || c == '\t' || c == '\n'; + if (wspace) { + was_wspace = true; + continue; + } + ++col; + if (was_wspace) { + if (col > 80) { + clean += '\n'; + col = 0; + } else if (!clean.empty()) + clean += ' '; + was_wspace = false; + } + clean += c; + } + return clean; + } + + + const std::string& html_; // no copy! + size_t pos_{}; + std::string raw_scraped_; +}; + + +G_GNUC_UNUSED static auto +format_as(const Context& ctx) +{ + return mu_format("<{}:{}: '{}'>", + ctx.position(), ctx.size(), + ctx.substr(ctx.position(), + std::min(static_cast<size_t>(8), + ctx.size() - ctx.position()))); +} + + +static void +skip_quoted(Context& ctx, std::string_view quote) +{ + while(!ctx.done()) { + if (ctx.looking_at(quote)) // closing quote + return; + ctx.advance(); + } +} + + +// attempt to skip over <script> / <style> blocks +static void +skip_script_style(Context& ctx, std::string_view tag) +{ + // <script> or <style> must be ignored + + bool escaped{}; + bool quoted{}, squoted{}; + bool inl_comment{}; + bool endl_comment{}; + + auto end_tag_str = mu_format("</{}>", tag); + auto end_tag = std::string_view(end_tag_str.data()); + + while (!ctx.done()) { + + if (inl_comment) { + if (ctx.looking_at("*/")) { + inl_comment = false; + ctx.advance(2); + } else + ctx.advance(); + continue; + } + + if (endl_comment) { + endl_comment = ctx.looking_at("\n"); + ctx.advance(); + continue; + } + + if (ctx.looking_at("\\")) { + escaped = !escaped; + ctx.advance(); + continue; + } + + if (ctx.looking_at("\"") && !escaped && squoted) { + quoted = !quoted; + ctx.advance(); + continue; + } + + if (ctx.looking_at("'") && !escaped && !quoted) { + squoted = !squoted; + ctx.advance(); + continue; + } + + + if (ctx.looking_at("/*")) { + inl_comment = true; + ctx.advance(2); + continue; + } + + if (ctx.looking_at("//")) { + endl_comment = true; + ctx.advance(2); + continue; + } + + if (!quoted && !squoted && ctx.looking_at(end_tag)) { + ctx.advance(end_tag.size()); + break; /* we're done, finally! */ + } + + ctx.advance(); + } +} + +// comment block; ignore completely +// pos will be immediately after the '<!-- +static void +comment(Context& ctx) +{ + constexpr std::string_view comment_endtag{"-->"}; + while (!ctx.done()) { + + if (ctx.looking_at(comment_endtag)) { + ctx.advance(comment_endtag.size()); + ctx.raw_scraped() += ' '; + return; + } + ctx.advance(); + } +} + +static bool // do we need a SPC separator for this tag? +needs_separator(std::string_view tagname) +{ + constexpr std::array<const char*, 7> nosep_tags = { + "b", "em", "i", "s", "strike", "tt", "u" + }; + return !seq_some(nosep_tags, [&](auto&& t){return matches(tagname, t);}); +} + +static bool // do we need to skip the element completely? +is_skip_element(std::string_view tagname) +{ + constexpr std::array<const char*, 4> skip_tags = { + "script", "style", "head", "meta" + }; + return seq_some(skip_tags, [&](auto&& t){return matches(tagname, t);}); +} + +// skip the end-tag +static void +end_tag(Context& ctx) +{ + while (!ctx.done()) { + if (ctx.looking_at(">")) { + ctx.advance(); + return; + } + ctx.advance(); + } +} + +// skip the whole element +static void +skip_element(Context& ctx, std::string_view tagname) +{ + // do something special? +} + + +// the start of a tag, i.e., pos will be just after the '<' +static void +tag(Context& ctx) +{ + // some elements we want to skip completely, + // for others just the tags. + constexpr std::string_view comment_start {"!--"}; + if (ctx.looking_at(comment_start)) { + ctx.advance(comment_start.size()); + comment(ctx); + return; + } + + if (ctx.looking_at("/")) { + ctx.advance(); + end_tag(ctx); + return; + } + + auto tagname = ctx.eat_head_word(); + if (tagname == "script" ||tagname == "style") { + skip_script_style(ctx, tagname); + return; + } + else if (is_skip_element(tagname)) + skip_element(ctx, tagname); + + const auto needs_sepa = needs_separator(tagname); + while (!ctx.done()) { + + if (ctx.looking_at("\"")) + skip_quoted(ctx, "\""); + + if (ctx.looking_at("'")) + skip_quoted(ctx, "'"); + + if (ctx.looking_at(">")) { + ctx.advance(); + if (needs_sepa) + ctx.raw_scraped() += ' '; + return; + } + ctx.advance(); + } +} + + +static void +html_escape_char(Context& ctx) +{ + // we only care about a few accented chars, and add them unaccented, lowercase, since that's + // we do for indexing anyway. + constexpr std::array<const char*, 11> escs = { + "breve", + "caron", + "circ", + "cute", + "grave", + "horn"/*thorn*/, + "macr", + "slash", + "strok", + "tilde", + "uml", + }; + + auto unescape=[escs](std::string_view esc)->char { + if (esc.empty()) + return ' '; + auto first{static_cast<char>(::tolower(esc.at(0)))}; + auto rest=esc.substr(1); + if (seq_some(escs, [&](auto&& e){return starts_with(rest, e);})) + return first; + else + return ' '; + }; + + size_t start_pos{ctx.position()}; + while (!ctx.done()) { + if (ctx.looking_at(";")) { + auto esc = ctx.substr(start_pos, ctx.position() - start_pos); + ctx.raw_scraped() += unescape(esc); + ctx.advance(); + return; + } + ctx.advance(); + } +} + + +// a block of text to be scraped +static void +text(Context& ctx) +{ + size_t start_pos{ctx.position()}; + while (!ctx.done()) { + + if (ctx.looking_at("&")) { + + ctx.raw_scraped() += ctx.substr(start_pos, + ctx.position() - start_pos); + ctx.advance(); + html_escape_char(ctx); + start_pos = ctx.position(); + + } else if (ctx.looking_at("<")) { + ctx.raw_scraped() += ctx.substr(start_pos, + ctx.position() - start_pos); + ctx.advance(); + tag(ctx); + start_pos = ctx.position(); + + } else + ctx.advance(); + } + + ctx.raw_scraped() += ctx.substr(start_pos, ctx.size() - start_pos); +} + +static Context *CTX{}; + +std::string +Mu::html_to_text(const std::string& html) +{ + Context ctx{html}; + CTX = &ctx; + + text(ctx); + + CTX = {}; + return ctx.scraped(); +} + +#ifdef BUILD_TESTS +#include "mu-test-utils.hh" + +static void +test_1() +{ + static std::vector<std::pair<std::string, std::string>> + tests = { + { "<!-- Hello -->A", "A" }, + { "A<!-- Test -->B", "A B" }, + { "A<i>a</i><b>p</b>", "Aap"}, + { "N&ocute;Ôt", "Noot"}, + { + "foo<!-- bar --><i>c</i>uu<bla>x</bla>" + "<!--hello -->world<!--", + "foo cuu x world" + } + }; + + for (auto&& test: tests) + assert_equal(html_to_text(test.first), test.second); +} + +static void +test_2() +{ + static std::vector<std::pair<std::string, std::string>> + tests = { + { R"(<i>hello, <b bar="/b">world!</b>)", + "hello, world!"}, + }; + + for (auto&& test: tests) + assert_equal(html_to_text(test.first), test.second); +} + + +static void +test_3() +{ + static std::vector<std::pair<std::string, std::string>> + tests = { + {R"(<i>hello, </i><script language="javascript"> + function foo() { + alert("Stroopwafel!"); // test + } + </script>world!)", + "hello, world!"}, + }; + + for (auto&& test: tests) + assert_equal(html_to_text(test.first), test.second); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/html-to-text/test-1", test_1); + g_test_add_func("/html-to-text/test-2", test_2); + g_test_add_func("/html-to-text/test-3", test_3); + + return g_test_run(); +} + + +#endif /*BUILD_TESTS*/ + + +#ifdef BUILD_HTML_TO_TEXT + +#include "mu-utils-file.hh" + +// simple tool that reads html on stdin and outputs text on stdout +// e.g. curl --silent https://www.example.com | build/lib/utils/mu-html2text + +int +main (int argc, char *argv[]) +{ + auto res = read_from_stdin(); + if (!res) { + mu_printerrln("error reading from stdin: {}", res.error().what()); + return 1; + } + + mu_println("{}", html_to_text(*res)); + + return 0; +} + +#endif /*BUILD_HTML_TO_TEXT*/ diff --git a/lib/utils/mu-lang-detector.cc b/lib/utils/mu-lang-detector.cc new file mode 100644 index 0000000..75af37e --- /dev/null +++ b/lib/utils/mu-lang-detector.cc @@ -0,0 +1,100 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" +#include "mu-lang-detector.hh" + +using namespace Mu; + +#ifndef HAVE_CLD2 +// Dummy implementation +Option<Language> Mu::detect_language(const std::string& txt) { return Nothing; } +#else +#include <cld2/public/compact_lang_det.h> +#include <cld2/public/encodings.h> + +Option<Language> +Mu::detect_language(const std::string& txt) +{ + bool is_reliable; + const auto lang = CLD2::DetectLanguage( + txt.c_str(), txt.length(), + true/*plain-text*/, + &is_reliable); + + if (lang == CLD2::UNKNOWN_LANGUAGE || !is_reliable) + return {}; + + Mu::Language res = { + CLD2::LanguageName(lang), + CLD2::LanguageCode(lang) + }; + if (!res.name || !res.code) + return {}; + else + return Some(std::move(res)); +} +#endif /*HAVE_CLD2*/ + +#ifdef BUILD_TESTS +#include <vector> +#include "mu-test-utils.hh" + +static void +test_lang_detector() +{ + using Case = std::tuple<std::string,std::string, std::string>; + using Cases = std::vector<Case>; + + const Cases tests = {{ + { "hello world, this is a bit of English", + "ENGLISH", "en" }, + { "En nu een paar Nederlandse woorden", + "DUTCH", "nl" }, + { "Hyvää huomenta! Puhun vähän suomea", + "FINNISH", "fi" }, + { "So eine Arbeit wird eigentlich nie fertig, man muß sie für " + "fertig erklären, wenn man nach Zeit und Umständen das " + "möglichste getan hat.", + "GERMAN", "de"} + }}; + + for (auto&& test: tests) { + const auto res = detect_language(std::get<0>(test)); +#ifndef HAVE_CLD2 + g_assert_false(!!res); +#else + g_assert_true(!!res); + assert_equal(std::get<1>(test), res->name); + assert_equal(std::get<2>(test), res->code); +#endif + + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/utils/lang-detector", test_lang_detector); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-lang-detector.hh b/lib/utils/mu-lang-detector.hh new file mode 100644 index 0000000..0b692bc --- /dev/null +++ b/lib/utils/mu-lang-detector.hh @@ -0,0 +1,46 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_LANG_DETECTOR_HH__ +#define MU_LANG_DETECTOR_HH__ + +#include <string> +#include "mu-option.hh" + +namespace Mu { + +struct Language { + const char *name; /**< Language name, e.g. "Dutch" */ + const char *code; /**< Language code, e.g. "nl" */ +}; + +/** + * Detect the language of text + * + * @param txt some text (UTF-8) + * + * @return either a Language or nothing; the latter + * also if we cannot not reliably determine a single language + */ +Option<Language> detect_language(const std::string& txt); + +} // namespace Mu + + +#endif /* MU_LANG_DETECTOR_HH__ */ diff --git a/lib/utils/mu-logger.cc b/lib/utils/mu-logger.cc new file mode 100644 index 0000000..c9f516d --- /dev/null +++ b/lib/utils/mu-logger.cc @@ -0,0 +1,239 @@ +/* +** Copyright (C) 2020-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#define G_LOG_USE_STRUCTURED +#include <glib.h> +#include <glib/gstdio.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include <iostream> +#include <fstream> +#include <cstring> + +#include <thread> +#include <mutex> + +#include "mu-logger.hh" + +using namespace Mu; + +static bool MuLogInitialized = false; +static Mu::Logger::Options MuLogOptions; +static std::ofstream MuStream; +static auto MaxLogFileSize = 1000 * 1024; +static std::mutex logger_mtx; + +static std::string MuLogPath; + +static bool +maybe_open_logfile() +{ + if (MuStream.is_open()) + return true; + + const auto logdir{to_string_gchar(g_path_get_dirname(MuLogPath.c_str()))}; + if (g_mkdir_with_parents(logdir.c_str(), 0700) != 0) { + mu_printerrln("creating {} failed: {}", logdir, g_strerror(errno)); + return false; + } + + MuStream.open(MuLogPath, std::ios::out | std::ios::app); + if (!MuStream.is_open()) { + mu_printerrln("opening {} failed: {}", MuLogPath, g_strerror(errno)); + return false; + } + + MuStream.sync_with_stdio(false); + return true; +} + +static bool +maybe_rotate_logfile() +{ + static unsigned n = 0; + + if (n++ % 1000 != 0) + return true; + + GStatBuf statbuf; + if (g_stat(MuLogPath.c_str(), &statbuf) == -1 || statbuf.st_size <= MaxLogFileSize) + return true; + + const auto old = MuLogPath + ".old"; + g_unlink(old.c_str()); // opportunistic + + if (MuStream.is_open()) + MuStream.close(); + + if (g_rename(MuLogPath.c_str(), old.c_str()) != 0) + mu_printerrln("failed to rename {} -> {}: {}", MuLogPath, old, g_strerror(errno)); + + return maybe_open_logfile(); +} + +static GLogWriterOutput +log_file(GLogLevelFlags level, const GLogField* fields, gsize n_fields, gpointer user_data) +{ + std::lock_guard lock{logger_mtx}; + + if (!maybe_open_logfile()) + return G_LOG_WRITER_UNHANDLED; + + char timebuf[22]; + time_t now{::time(NULL)}; + ::strftime(timebuf, sizeof(timebuf), "%F %T", ::localtime(&now)); + + char* msg = g_log_writer_format_fields(level, fields, n_fields, FALSE); + if (msg && msg[0] == '\n') // hmm... seems lines start with '\n'r + msg[0] = ' '; + + MuStream << timebuf << ' ' << msg << std::endl; + + g_free(msg); + + return maybe_rotate_logfile() ? G_LOG_WRITER_HANDLED : G_LOG_WRITER_UNHANDLED; +} + +static GLogWriterOutput +log_stdouterr(GLogLevelFlags level, const GLogField* fields, gsize n_fields, gpointer user_data) +{ + return g_log_writer_standard_streams(level, fields, n_fields, user_data); +} + +static GLogWriterOutput +log_journal(GLogLevelFlags level, const GLogField* fields, gsize n_fields, gpointer user_data) +{ + return g_log_writer_journald(level, fields, n_fields, user_data); +} + + +Result<Logger> +Mu::Logger::make(const std::string& path, Mu::Logger::Options opts) +{ + if (MuLogInitialized) + return Err(Error::Code::Internal, "logging already initialized"); + + return Ok(Logger(path, opts)); +} + +Mu::Logger::Logger(const std::string& path, Mu::Logger::Options opts) +{ + if (g_getenv("MU_LOG_STDOUTERR")) + opts |= Logger::Options::StdOutErr; + + MuLogOptions = opts; + MuLogPath = path; + + g_log_set_writer_func( + [](GLogLevelFlags level, const GLogField* fields, gsize n_fields, gpointer user_data) { + // filter out debug-level messages? + if (level == G_LOG_LEVEL_DEBUG && + (none_of(MuLogOptions & Options::Debug))) + return G_LOG_WRITER_HANDLED; + + // log criticals to stdout / err or if asked + if (level == G_LOG_LEVEL_CRITICAL || + any_of(MuLogOptions & Options::StdOutErr)) { + log_stdouterr(level, fields, n_fields, user_data); + } + + // log to the journal, or, if not available to a file. + if (any_of(MuLogOptions & Options::File) || + log_journal(level, fields, n_fields, user_data) != G_LOG_WRITER_HANDLED) + return log_file(level, fields, n_fields, user_data); + else + return G_LOG_WRITER_HANDLED; + }, + NULL, + NULL); + + g_message("logging initialized; debug: %s, stdout/stderr: %s", + any_of(opts & Options::Debug) ? "yes" : "no", + any_of(opts & Options::StdOutErr) ? "yes" : "no"); + + MuLogInitialized = true; +} + +Logger::~Logger() +{ + if (!MuLogInitialized) + return; + + if (MuStream.is_open()) + MuStream.close(); + + MuLogInitialized = false; +} + + +#ifdef BUILD_TESTS +#include <vector> +#include <atomic> + +#include "mu-test-utils.hh" +#include "mu-utils-file.hh" + +static void +test_logger_threads(void) +{ + TempDir temp_dir; + const auto testpath{join_paths(temp_dir.path(), "test.log")}; + mu_message("log-file: {}", testpath); + + auto logger = Logger::make(testpath, Logger::Options::File | Logger::Options::Debug); + assert_valid_result(logger); + + const auto thread_num = 16; + std::atomic<bool> running = true; + + std::vector<std::thread> threads; + + /* log to the logger file from many threass */ + for (auto n = 0; n != thread_num; ++n) + threads.emplace_back( + std::thread([&running]{ + while (running) { + //mu_debug("log message from thread <{}>", n); + std::this_thread::yield(); + } + })); + + using namespace std::chrono_literals; + std::this_thread::sleep_for(1s); + running = false; + + for (auto n = 0; n != 16; ++n) + if (threads[n].joinable()) + threads[n].join(); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/utils/logger", test_logger_threads); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-logger.hh b/lib/utils/mu-logger.hh new file mode 100644 index 0000000..6024e28 --- /dev/null +++ b/lib/utils/mu-logger.hh @@ -0,0 +1,74 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_LOGGER_HH__ +#define MU_LOGGER_HH__ + +#include <string> +#include <utils/mu-utils.hh> +#include <utils/mu-result.hh> + +namespace Mu { + +/** + * RAII object for handling logging (through g_(debug|warning|...)) + * + */ +struct Logger { + + /** + * Logging options + * + */ + enum struct Options { + None = 0, /**< Nothing specific */ + StdOutErr = 1 << 1, /**< Log to stdout/stderr */ + File = 1 << 2, /**< Force logging to file, even if journal available */ + Debug = 1 << 3, /**< Include debug-level logs */ + }; + + /** + * Initialize the logging sub-system. + * + * Note that the path is only used if structured logging fails -- + * practically, it goes to the file if there's no systemd/journald. + * + * if the environment variable MU_LOG_STDOUTERR is set, + * LogOptions::StdoutErr is implied. + * + * @param path path to the log file + * @param opts logging options + */ + static Result<Logger> make(const std::string& path, Options opts=Options::None); + + /** + * DTOR + * + */ + ~Logger(); + +private: + Logger(const std::string& path, Options opts); +}; + +MU_ENABLE_BITOPS(Logger::Options); + +} // namespace Mu + +#endif /* MU_LOGGER_HH__ */ diff --git a/lib/utils/mu-option.cc b/lib/utils/mu-option.cc new file mode 100644 index 0000000..e096117 --- /dev/null +++ b/lib/utils/mu-option.cc @@ -0,0 +1,106 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-option.hh" +#include <glib.h> + +using namespace Mu; + +Mu::Option<std::string> +Mu::to_string_opt_gchar(gchar*&& str) +{ + auto res = to_string_opt(str); + g_free(str); + + return res; +} + +#if BUILD_TESTS +#include "mu-test-utils.hh" + +static Option<int> +get_opt_int(bool b) +{ + if (b) + return Some(123); + else + return Nothing; +} + +static void +test_option() +{ + { + const auto oi{get_opt_int(true)}; + g_assert_true(!!oi); + g_assert_cmpint(oi.value(), ==, 123); + } + + { + const auto oi{get_opt_int(false)}; + g_assert_false(!!oi); + g_assert_false(oi.has_value()); + g_assert_cmpint(oi.value_or(456), ==, 456); + } +} + +static void +test_unwrap() +{ + { + auto&& oi{get_opt_int(true)}; + g_assert_cmpint(unwrap(std::move(oi)), ==, 123); + } + + auto ex{0}; + try { + auto&& oi{get_opt_int(false)}; + unwrap(std::move(oi)); + } catch(...) { + ex = 1; + } + + g_assert_cmpuint(ex, ==, 1); +} + +static void +test_opt_gchar() +{ + auto o1{to_string_opt_gchar(g_strdup("boo!"))}; + auto o2{to_string_opt_gchar(nullptr)}; + + g_assert_false(!!o2); + g_assert_true(o1.value() == "boo!"); +} + + + +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/option/option", test_option); + g_test_add_func("/option/unwrap", test_unwrap); + g_test_add_func("/option/opt-gchar", test_opt_gchar); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-option.hh b/lib/utils/mu-option.hh new file mode 100644 index 0000000..32b1bee --- /dev/null +++ b/lib/utils/mu-option.hh @@ -0,0 +1,77 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_OPTION__ +#define MU_OPTION__ + +#include <tl/optional.hpp> +#include <stdexcept> +#include <string> + +namespace Mu { + +/// Either a value of type T, or None +template <typename T> using Option = tl::optional<T>; + +template <typename T> +Option<T> +Some(T&& t) +{ + return std::move(t); +} +constexpr auto Nothing = tl::nullopt; // 'None' is already taken. + +template<typename T> T +unwrap(Option<T>&& res) +{ + if (!!res) + return std::move(res.value()); + else + throw std::runtime_error("failure is not an option"); +} + + +/** + * Maybe create a string from a const char pointer. + * + * @param str a char pointer or NULL + * + * @return option with either the string or nothing if str was NULL. + */ +Option<std::string> +static inline to_string_opt(const char* str) { + if (str) + return std::string{str}; + else + return Nothing; +} + +/** + * Like maybe_string that takes a const char*, but additionally, + * g_free() the string. + * + * @param str char pointer or NULL (consumed) + * + * @return option with either the string or nothing if str was NULL. + */ +Option<std::string> to_string_opt_gchar(char*&& str); + + +} // namespace Mu +#endif /*MU_OPTION__*/ diff --git a/lib/utils/mu-readline.cc b/lib/utils/mu-readline.cc new file mode 100644 index 0000000..edf6a52 --- /dev/null +++ b/lib/utils/mu-readline.cc @@ -0,0 +1,136 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "mu-utils.hh" +#include "mu-readline.hh" + +#include <string> +#include <unistd.h> + +#ifdef HAVE_LIBREADLINE +#if defined(HAVE_READLINE_READLINE_H) +#include <readline/readline.h> +#elif defined(HAVE_READLINE_H) +#include <readline.h> +#else /* !defined(HAVE_READLINE_H) */ +extern char* readline(); +#endif /* !defined(HAVE_READLINE_H) */ +char* cmdline = NULL; +#else /* !defined(HAVE_READLINE_READLINE_H) */ +/* no readline */ +#endif /* HAVE_LIBREADLINE */ + +#ifdef HAVE_READLINE_HISTORY +#if defined(HAVE_READLINE_HISTORY_H) +#include <readline/history.h> +#elif defined(HAVE_HISTORY_H) +#include <history.h> +#else /* !defined(HAVE_HISTORY_H) */ +extern void add_history(); +extern int write_history(); +extern int read_history(); +#endif /* defined(HAVE_READLINE_HISTORY_H) */ +/* no history */ +#endif /* HAVE_READLINE_HISTORY */ + +#if defined(HAVE_LIBREADLINE) && defined(HAVE_READLINE_HISTORY) +#define HAVE_READLINE (1) +#else +#define HAVE_READLINE (0) +#endif + +using namespace Mu; + +static bool is_a_tty{}; +static std::string hist_path; +static size_t max_lines{}; + +// LCOV_EXCL_START + +bool +Mu::have_readline() +{ + return HAVE_READLINE != 0; +} + +void +Mu::setup_readline(const std::string& histpath, size_t maxlines) +{ + is_a_tty = !!::isatty(::fileno(stdout)); + hist_path = histpath; + max_lines = maxlines; + +#if HAVE_READLINE + rl_bind_key('\t', rl_insert); // default (filenames) is not useful + using_history(); + read_history(hist_path.c_str()); + + if (max_lines > 0) + stifle_history(max_lines); +#endif /*HAVE_READLINE*/ +} + +void +Mu::shutdown_readline() +{ +#if HAVE_READLINE + if (!is_a_tty) + return; + + write_history(hist_path.c_str()); + if (max_lines > 0) + history_truncate_file(hist_path.c_str(), max_lines); +#endif /*HAVE_READLINE*/ +} + +std::string +Mu::read_line(bool& do_quit) +{ +#if HAVE_READLINE + if (is_a_tty) { + auto buf = readline(";; mu% "); + if (!buf) { + do_quit = true; + return {}; + } + std::string line{buf}; + ::free(buf); + return line; + } +#endif /*HAVE_READLINE*/ + + std::string line; + mu_print(";; mu> "); + if (!std::getline(std::cin, line)) + do_quit = true; + + return line; +} + +void +Mu::save_line(const std::string& line) +{ +#if HAVE_READLINE + if (is_a_tty) + add_history(line.c_str()); +#endif /*HAVE_READLINE*/ +} + +// LCOV_EXCL_STOP diff --git a/lib/utils/mu-readline.hh b/lib/utils/mu-readline.hh new file mode 100644 index 0000000..ca0455f --- /dev/null +++ b/lib/utils/mu-readline.hh @@ -0,0 +1,61 @@ +/* +** Copyright (C) 2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <string> + +namespace Mu { + +/** + * Setup readline when available and on tty. + * + * @param histpath path to the history file + * @param max_lines maximum number of history to save + */ +void setup_readline(const std::string& histpath, size_t max_lines); + +/** + * Shutdown readline + * + */ +void shutdown_readline(); + +/** + * Read a command line + * + * @param do_quit recceives whether we should quit. + * + * @return the string read or empty + */ +std::string read_line(bool& do_quit); + +/** + * Save a line to history (or do nothing when readline is not active) + * + * @param line a line. + */ +void save_line(const std::string& line); + + +/** + * Do we have the non-shim readline? + * + * @return true or failse + */ +bool have_readline(); + +} // namespace Mu diff --git a/lib/utils/mu-regex.cc b/lib/utils/mu-regex.cc new file mode 100644 index 0000000..8127695 --- /dev/null +++ b/lib/utils/mu-regex.cc @@ -0,0 +1,114 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-regex.hh" +#include <iostream> + +using namespace Mu; + +#if BUILD_TESTS +#include "mu-test-utils.hh" + +// No need for extensive regex test, we just rely on GRegex. + +static void +test_regex_match() +{ + auto rx = Regex::make("a.*b.c"); + assert_valid_result(rx); + + assert_equal(mu_format("{}", *rx), "/a.*b.c/"); + + g_assert_true(rx->matches("axxxxxbqc")); + g_assert_false(rx->matches("axxxxxbqqc")); + + { // unset matches nothing. + Regex rx2; + g_assert_false(rx2.matches("")); + } +} + + +static void +test_regex_match2() +{ + Regex rx; + { + std::string foo = "h.llo"; + rx = unwrap(Regex::make(foo.c_str())); + } + + std::string hei = "hei"; + + g_assert_true(rx.matches("hallo")); + g_assert_false(rx.matches(hei)); +} + + +static void +test_regex_replace() +{ + { + auto rx = Regex::make("f.o"); + assert_valid_result(rx); + assert_equal(rx->replace("foobar", "cuux").value_or("error"), "cuuxbar"); + } + + { + auto rx = Regex::make("f.o", G_REGEX_MULTILINE); + assert_valid_result(rx); + assert_equal(rx->replace("foobar\nfoobar", "cuux").value_or("error"), + "cuuxbar\ncuuxbar"); + } +} + + +static void +test_regex_fail() +{ + allow_warnings(); + + { // unset rx can't replace / error. + Regex rx; + assert_equal(mu_format("{}", rx), "//"); + g_assert_false(!!rx.replace("foo", "bar")); + } + + { + auto rx = Regex::make("("); + g_assert_false(!!rx); + + } + +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/regex/match", test_regex_match); + g_test_add_func("/regex/match2", test_regex_match2); + g_test_add_func("/regex/replace", test_regex_replace); + g_test_add_func("/regex/fail", test_regex_fail); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-regex.hh b/lib/utils/mu-regex.hh new file mode 100644 index 0000000..c5fd4a0 --- /dev/null +++ b/lib/utils/mu-regex.hh @@ -0,0 +1,193 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#ifndef MU_REGEX_HH__ +#define MU_REGEX_HH__ + +#include <glib.h> +# +#include <utils/mu-result.hh> +#include <utils/mu-utils.hh> + +namespace Mu { +/** + * RAII wrapper around a GRegex which in itself is a wrapper around PCRE. We use + * PCRE rather than std::regex because it is much faster. + */ +struct Regex { +#if !GLIB_CHECK_VERSION(2,74,0) /* backward compat */ +#define G_REGEX_DEFAULT (static_cast<GRegexCompileFlags>(0)) +#define G_REGEX_MATCH_DEFAULT (static_cast<GRegexMatchFlags>(0)) +#endif + /** + * Trivial constructor + * + * @return + */ + Regex() noexcept: rx_{} {} + + /** + * Construct a new Regex object + * + * @param ptrn PRRE regular expression pattern + * @param cflags compile flags + * @param mflags match flags + * + * @return a Regex object or an error. + */ + static Result<Regex> make(const std::string& ptrn, + GRegexCompileFlags cflags = G_REGEX_DEFAULT, + GRegexMatchFlags mflags = G_REGEX_MATCH_DEFAULT) noexcept try { + return Regex(ptrn.c_str(), cflags, mflags); + } catch (const Error& err) { + return Err(err); + } + + + /** + * Copy CTOR + * + * @param other some other Regex + */ + Regex(const Regex& other) noexcept: rx_{} { *this = other; } + + /** + * Move CTOR + * + * @param other some other Regex + */ + Regex(Regex&& other) noexcept: rx_{} { *this = std::move(other); } + + + /** + * DTOR + */ + ~Regex() noexcept { g_clear_pointer(&rx_, g_regex_unref); } + + /** + * Cast to the the underlying GRegex* + * + * @return a GRegex* + */ + operator const GRegex*() const noexcept { return rx_; } + + /** + * Doe this object contain a valid GRegex*? + * + * @return true or false + */ + operator bool() const noexcept { return !!rx_; } + + /** + * operator= + * + * @param other copy some other object to this one + * + * @return *this + */ + Regex& operator=(const Regex& other) noexcept { + if (this != &other) { + g_clear_pointer(&rx_, g_regex_unref); + if (other.rx_) + rx_ = g_regex_ref(other.rx_); + } + return *this; + } + + /** + * operator= + * + * @param other move some other object to this one + * + * @return *this + */ + Regex& operator=(Regex&& other) noexcept { + if (this != &other) { + g_clear_pointer(&rx_, g_regex_unref); + rx_ = other.rx_; + other.rx_ = nullptr; + } + return *this; + } + + /** + * Does this regexp match the given string? An unset Regex matches + * nothing. + * + * @param str string to test + * @param mflags match flags + * + * @return true or false + */ + bool matches(const std::string& str, + GRegexMatchFlags mflags=G_REGEX_MATCH_DEFAULT) const noexcept { + if (!rx_) + return false; + else + return g_regex_match(rx_, str.c_str(), mflags, nullptr); + // strangely, valgrind reports some memory error related to + // the str.c_str(). It *seems* like a false alarm. + } + + /** + * Replace all occurrences of @this regexp in some string with a + * replacement string + * + * @param str some string + * @param repl replacement string + * + * @return string or error + */ + Result<std::string> replace(const std::string& str, const std::string& repl) const { + GError *gerr{}; + + if (!rx_) + return Err(Error::Code::InvalidArgument, "missing regexp"); + else if (auto&& s{g_regex_replace(rx_, str.c_str(), str.length(), 0, + repl.c_str(), G_REGEX_MATCH_DEFAULT, &gerr)}; !s) + return Err(Error::Code::InvalidArgument, &gerr, "error in Regex::replace"); + else + return Ok(to_string_gchar(std::move(s))); + } + + const GRegex* g_regex() const { return rx_; } + +private: + Regex(const char *ptrn, GRegexCompileFlags cflags, GRegexMatchFlags mflags) { + GError *err{}; + if (rx_ = g_regex_new(ptrn, cflags, mflags, &err); !rx_) + throw Error{Error::Code::InvalidArgument, &err, + "invalid regexp: '{}'", ptrn}; + } + + GRegex *rx_{}; +}; + +static inline std::string format_as(const Regex& rx) { + if (auto&& grx{rx.g_regex()}; !grx) + return "//"; + else + return mu_format("/{}/", g_regex_get_pattern(grx)); + +} + + +} // namespace Mu + + +#endif /* MU_REGEX_HH__ */ diff --git a/lib/utils/mu-result.hh b/lib/utils/mu-result.hh new file mode 100644 index 0000000..887f8ad --- /dev/null +++ b/lib/utils/mu-result.hh @@ -0,0 +1,145 @@ +/* +** Copyright (C) 2019-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_RESULT_HH__ +#define MU_RESULT_HH__ + +#include <tl/expected.hpp> +#include "utils/mu-error.hh" + +namespace Mu { +/** + * A little Rust-envy...a Result is _either_ some value of type T, _or_ a Mu::Error + */ +template <typename T> using Result = tl::expected<T, Error>; + +/** + * Ok() is not typically strictly needed (unlike Err), but imitates Rust's Ok + * and it helps the reader. + * + * @param t the value to return + * + * @return a success Result<T> + */ +template <typename T> Result<T> +Ok(T&& t) +{ + return std::move(t); +} + +/** + * Implementation of Ok() for void results. + * + * @return a success Result<void> + */ +static inline Result<void> +Ok() +{ + return {}; +} + +/** + * Return an error + * + * @param err the error + * + * @return error + */ +template<typename T> Result<T> +Err(Error&& err) +{ + return tl::unexpected(std::move(err)); +} +template<typename T> Result<T> +Err(const Error& err) +{ + return tl::unexpected(err); +} + +static inline tl::unexpected<Error> +Err(Error&& err) +{ + return tl::unexpected(std::move(err)); +} + +static inline tl::unexpected<Error> +Err(const Error& err) +{ + return tl::unexpected(err); +} + +template<typename T> +static inline tl::unexpected<Error> +Err(const Result<T>& res) +{ + return res.error(); +} + +template<typename T> +static inline tl::unexpected<Error> +Err(Result<T>&& res) +{ + return std::move(res.error()); +} + +/* + * convenience + */ +template <typename ...T> +tl::unexpected<Error> +Err(Error::Code code, fmt::format_string<T...> frm, T&&... args) +{ + return Err(Error{code, frm, std::forward<T>(args)...}); +} + +template <typename ...T> +tl::unexpected<Error> +Err(Error::Code code, GError **err, fmt::format_string<T...> frm, T&&... args) +{ + return Err(Error{code, err, frm, std::forward<T>(args)...}); +} + + +template<typename T> T +unwrap(Result<T>&& res) +{ + if (!!res) + return std::move(res.value()); + else + throw res.error(); +} + +/** + * Assert that some result has a value (for unit tests) + * + * @param R some result + */ +#define assert_valid_result(R) do { \ + auto&& res__ = R; \ + if(!res__) { \ + mu_printerrln("{}:{}: error-result: {}", \ + __FILE__, __LINE__, \ + (res__).error().what()); \ + g_assert_true(!!res__); \ + } \ +} while(0) + +}// namespace Mu + +#endif /* MU_RESULT_HH__ */ diff --git a/lib/utils/mu-sexp.cc b/lib/utils/mu-sexp.cc new file mode 100644 index 0000000..47510d1 --- /dev/null +++ b/lib/utils/mu-sexp.cc @@ -0,0 +1,522 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#include "mu-sexp.hh" +#include "mu-utils.hh" + +#include <atomic> +#include <sstream> +#include <array> + +using namespace Mu; + +template<typename...T> static Mu::Error +parsing_error(size_t pos, fmt::format_string<T...> frm, T&&... args) +{ + const auto&& msg{fmt::format(frm, std::forward<T>(args)...)}; + if (pos == 0) + return Mu::Error(Error::Code::Parsing, "{}", msg); + else + return Mu::Error(Error::Code::Parsing, "{}: {}", pos, msg); +} + +static size_t +skip_whitespace(const std::string& s, size_t pos) +{ + while (pos != s.size()) { + if (s[pos] == ' ' || s[pos] == '\t' || s[pos] == '\n') + ++pos; + else + break; + } + return pos; +} + +static Result<Sexp> parse(const std::string& expr, size_t& pos); + +static Result<Sexp> +parse_list(const std::string& expr, size_t& pos) +{ + if (expr[pos] != '(') // sanity check. + return Err(parsing_error(pos, "expected: '(' but got '{}", expr[pos])); + + Sexp lst{}; + + ++pos; + while (expr[pos] != ')' && pos != expr.size()) { + if (auto&& item = parse(expr, pos); item) + lst.add(std::move(*item)); + else + return Err(item.error()); + } + + if (expr[pos] != ')') + return Err(parsing_error(pos, "expected: ')' but got '{}'", expr[pos])); + ++pos; + return Ok(std::move(lst)); +} + +static Result<Sexp> +parse_string(const std::string& expr, size_t& pos) +{ + if (expr[pos] != '"') // sanity check. + return Err(parsing_error(pos, "expected: '\"'' but got '{}", expr[pos])); + + bool escape{}; + std::string str; + for (++pos; pos != expr.size(); ++pos) { + auto kar = expr[pos]; + if (escape && (kar == '"' || kar == '\\')) { + str += kar; + escape = false; + continue; + } + + if (kar == '"') + break; + else if (kar == '\\') + escape = true; + else + str += kar; + } + + if (escape || expr[pos] != '"') + return Err(parsing_error(pos, "unterminated string '{}'", str)); + + ++pos; + return Ok(Sexp{std::move(str)}); +} + + +static Result<Sexp> +parse_integer(const std::string& expr, size_t& pos) +{ + if (!isdigit(expr[pos]) && expr[pos] != '-') // sanity check. + return Err(parsing_error(pos, "expected: <digit> but got '{}", expr[pos])); + + std::string num; // negative number? + if (expr[pos] == '-') { + num = "-"; + ++pos; + } + + for (; isdigit(expr[pos]); ++pos) + num += expr[pos]; + + return Ok(Sexp{::atoi(num.c_str())}); +} + +static Result<Sexp> +parse_symbol(const std::string& expr, size_t& pos) +{ + if (!isalpha(expr[pos]) && expr[pos] != ':') // sanity check. + return Err(parsing_error(pos, "expected: <alpha>|: but got '{}", expr[pos])); + + std::string symb(1, expr[pos]); + for (++pos; isalnum(expr[pos]) || expr[pos] == '-'; ++pos) + symb += expr[pos]; + + return Ok(Sexp{Sexp::Symbol{symb}}); +} + +static Result<Sexp> +parse(const std::string& expr, size_t& pos) +{ + pos = skip_whitespace(expr, pos); + + if (pos == expr.size()) + return Err(parsing_error(pos, "expected: character '{}", expr[pos])); + + const auto kar = expr[pos]; + const auto sexp = std::invoke([&]() -> Result<Sexp> { + if (kar == '(') + return parse_list(expr, pos); + else if (kar == '"') + return parse_string(expr, pos); + else if (isdigit(kar) || kar == '-') + return parse_integer(expr, pos); + else if (isalpha(kar) || kar == ':') + return parse_symbol(expr, pos); + else + return Err(parsing_error(pos, "unexpected character '{}", kar)); + }); + + if (sexp) + pos = skip_whitespace(expr, pos); + + return sexp; +} + +Result<Sexp> +Sexp::parse(const std::string& expr) +{ + size_t pos{}; + auto res = ::parse(expr, pos); + if (!res) + return res; + else if (pos != expr.size()) + return Err(parsing_error(pos, "trailing data starting with '{}'", expr[pos])); + else + return res; +} + +std::string +Sexp::to_string(Format fopts) const +{ + std::stringstream sstrm; + const auto splitp{any_of(fopts & Format::SplitList)}; + const auto typeinfop{any_of(fopts & Format::TypeInfo)}; + + if (listp()) { + sstrm << '('; + bool first{true}; + for(auto&& elm: list()) { + sstrm << (first ? "" : " ") << elm.to_string(fopts); + first = false; + } + sstrm << ')'; + if (splitp) + sstrm << '\n'; + } else if (stringp()) + sstrm << quote(string()); + else if (numberp()) + sstrm << number(); + else if (symbolp()) + sstrm << symbol().name; + + if (typeinfop) + sstrm << '<' << Sexp::type_name(type()) << '>'; + + return sstrm.str(); +} + +// LCOV_EXCL_START + +std::string +Sexp::to_json_string(Format fopts) const +{ + std::stringstream sstrm; + + switch (type()) { + case Type::List: { + // property-lists become JSON objects + if (plistp()) { + sstrm << "{"; + auto it{list().begin()}; + bool first{true}; + while (it != list().end()) { + sstrm << (first ? "" : ",") << quote(it->symbol().name) << ":"; + ++it; + sstrm << it->to_json_string(); + ++it; + first = false; + } + sstrm << "}"; + if (any_of(fopts & Format::SplitList)) + sstrm << '\n'; + } else { // other lists become arrays. + sstrm << '['; + bool first{true}; + for (auto&& child : list()) { + sstrm << (first ? "" : ", ") << child.to_json_string(); + first = false; + } + sstrm << ']'; + if (any_of(fopts & Format::SplitList)) + sstrm << '\n'; + } + break; + } + case Type::String: + sstrm << quote(string()); + break; + case Type::Symbol: + if (nilp()) + sstrm << "false"; + else if (symbol() == "t") + sstrm << "true"; + else + sstrm << quote(symbol().name); + break; + case Type::Number: + sstrm << number(); + break; + default: + break; + } + + return sstrm.str(); +} + + + +Sexp& +Sexp::del_prop(const std::string& pname) +{ + if (auto kill_it = find_prop(pname, begin(), end()); kill_it != cend()) + list().erase(kill_it, kill_it + 2); + return *this; +} + + +Sexp::const_iterator +Sexp::find_prop(const std::string& s, + Sexp::const_iterator b, Sexp::const_iterator e) const +{ + for (auto&& it = b; it != e && it+1 != e; it += 2) + if (it->symbolp() && it->symbol() == s) + return it; + return e; +} + +Sexp::iterator +Sexp::find_prop(const std::string& s, + Sexp::iterator b, Sexp::iterator e) +{ + for (auto&& it = b; it != e && it+1 != e; it += 2) + if (it->symbolp() && it->symbol() == s) + return it; + return e; +} + + +bool +Sexp::plistp(Sexp::const_iterator b, Sexp::const_iterator e) const +{ + if (b == e) + return true; + else if (b + 1 == e) + return false; + else + return b->symbolp() && plistp(b + 2, e); +} + + +// LCOV_EXCL_STOP + +#if BUILD_TESTS + +#include "mu-test-utils.hh" + +static void +test_list() +{ + { + Sexp s; + g_assert_true(s.listp()); + g_assert_true(s.to_string() == "()"); + g_assert_true(Sexp::type_name(s.type()) == "list"); + g_assert_true(s.empty()); + } + + { + Sexp::List items = { + Sexp("hello"), + Sexp(123), + Sexp::Symbol("world") + }; + const Sexp s{std::move(items)}; + g_assert_false(s.empty()); + g_assert_cmpuint(s.size(),==,3); + g_assert_true(s.to_string() == "(\"hello\" 123 world)"); + + + /* copy */ + Sexp s2 = s; + g_assert_true(s2.to_string() == "(\"hello\" 123 world)"); + + /* move */ + Sexp s3 = std::move(s2); + g_assert_true(s3.to_string() == "(\"hello\" 123 world)"); + + s3.clear(); + g_assert_true(s3.empty()); + } + +} + +static void +test_string() +{ + { + Sexp s("hello"); + g_assert_true(s.stringp()); + g_assert_true(s.string()=="hello"); + g_assert_true(s.to_string()=="\"hello\""); + g_assert_true(Sexp::type_name(s.type()) == "string"); + } + + { + // Sexp s(std::string_view("hel\"lo")); + // g_assert_true(s.is_string()); + // g_assert_cmpstr(s.string().c_str(),==,"hel\"lo"); + // g_assert_cmpstr(s.to_string().c_str(),==,"\"hel\\\"lo\""); + } +} + +static void +test_number() +{ + { + Sexp s(123); + g_assert_true(s.numberp()); + g_assert_cmpint(s.number(),==,123); + g_assert_true(s.to_string() == "123"); + g_assert_true(Sexp::type_name(s.type()) == "number"); + } + + { + Sexp s(true); + g_assert_true(s.numberp()); + g_assert_cmpint(s.number(),==,1); + g_assert_true(s.to_string()=="1"); + } +} + +static void +test_symbol() +{ + { + Sexp s{Sexp::Symbol("hello")}; + g_assert_true(s.symbolp()); + g_assert_true(s.symbol()=="hello"); + g_assert_true (s.to_string()=="hello"); + g_assert_true(Sexp::type_name(s.type()) == "symbol"); + } + + { + Sexp s{"hello"_sym}; + g_assert_true(s.symbolp()); + g_assert_true(s.symbol()=="hello"); + g_assert_true (s.to_string()=="hello"); + } + +} + +static void +test_multi() +{ + Sexp s{"abc", 123, Sexp::Symbol{"def"}}; + g_assert_true(s.to_string() == "(\"abc\" 123 def)"); +} + + +static void +test_add() +{ + { + Sexp s{"abc", 123}; + s.add("def"_sym); + g_assert_true(s.to_string() == "(\"abc\" 123 def)"); + } +} + +static void +test_add_multi() +{ + { + Sexp s{"abc", 123}; + s.add("def"_sym, 456, Sexp{"boo", 2}); + g_assert_true(s.to_string() == "(\"abc\" 123 def 456 (\"boo\" 2))"); + } + + { + Sexp s{"abc", 123}; + Sexp t{"boo", 2}; + s.add("def"_sym, 456, t); + g_assert_true(s.to_string() == "(\"abc\" 123 def 456 (\"boo\" 2))"); + } + +} + +static void +test_plist() +{ + Sexp s; + s.put_props("hello", "world"_sym, "foo", 123, "bar"_sym, "cuux"); + g_assert_true(s.to_string() == R"((hello world foo 123 bar "cuux"))"); + + s.put_props("hello", 12345); + g_assert_true(s.to_string() == R"((foo 123 bar "cuux" hello 12345))"); +} + + +static void +check_parse(const std::string& expr, const std::string& expected) +{ + auto sexp = Sexp::parse(expr); + assert_valid_result(sexp); + assert_equal(to_string(*sexp), expected); +} + +static void +test_parser() +{ + check_parse(":foo-123", ":foo-123"); + check_parse("foo", "foo"); + check_parse(R"(12345)", "12345"); + check_parse(R"(-12345)", "-12345"); + check_parse(R"((123 bar "cuux"))", "(123 bar \"cuux\")"); + + check_parse(R"("foo\"bar\"cuux")", "\"foo\\\"bar\\\"cuux\""); + + check_parse(R"("foo +bar")", + "\"foo\nbar\""); +} + +static void +test_parser_fail() +{ + g_assert_false(!!Sexp::parse("\"")); + g_assert_false(!!Sexp::parse("123abc")); + g_assert_false(!!Sexp::parse("(")); + g_assert_false(!!Sexp::parse(")")); + g_assert_false(!!Sexp::parse("(hello (boo))))")); + + g_assert_true(Sexp::type_name(static_cast<Sexp::Type>(-1)) == "<error>"); +} + + +int +main(int argc, char* argv[]) +try { + mu_test_init(&argc, &argv); + + g_test_add_func("/sexp/list", test_list); + g_test_add_func("/sexp/string", test_string); + g_test_add_func("/sexp/number", test_number); + g_test_add_func("/sexp/symbol", test_symbol); + g_test_add_func("/sexp/multi", test_multi); + g_test_add_func("/sexp/add", test_add); + g_test_add_func("/sexp/add-multi", test_add_multi); + g_test_add_func("/sexp/plist", test_plist); + g_test_add_func("/sexp/parser", test_parser); + g_test_add_func("/sexp/parser-fail", test_parser_fail); + + return g_test_run(); + +} catch (const std::runtime_error& re) { + mu_printerrln("{}", re.what()); + return 1; +} + + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-sexp.hh b/lib/utils/mu-sexp.hh new file mode 100644 index 0000000..8127cbf --- /dev/null +++ b/lib/utils/mu-sexp.hh @@ -0,0 +1,326 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_SEXP_HH__ +#define MU_SEXP_HH__ + +#include "mu-utils.hh" + +#include <stdexcept> +#include <vector> +#include <string> +#include <string_view> +#include <iostream> +#include <variant> +#include <cinttypes> +#include <ostream> +#include <cassert> + +#include <utils/mu-result.hh> +#include <utils/mu-option.hh> + +namespace Mu { + +/** + * A structure somewhat similar to a Lisp s-expression and which can be + * constructed from/to an s-expressing string representation. + * + * A sexp is either an atom (String, Number, Symbol) or a List. + */ +struct Sexp { + /** + * Types + * + */ + using List = std::vector<Sexp>; + using String = std::string; + using Number = int64_t; + struct Symbol { // distinguish from String. + Symbol(const std::string& s): name{s} {} + Symbol(std::string&& s): name(std::move(s)) {} + Symbol(const char* str): Symbol(std::string{str}) {} + Symbol(std::string_view sv): Symbol(std::string{sv}) {} + operator const std::string&() const {return name; } + std::string name; + + bool operator==(const Symbol& rhs) const { + return this == &rhs ? true : rhs.name == name; + } + bool operator!=(const Symbol& rhs) const { return *this == rhs ? false : true; } + }; + enum struct Type { List, String, Number, Symbol }; + using ValueType = std::variant<List, String, Number, Symbol>; + + /** + * Is some Sexp of the given type? + * + * @return true or false + */ + constexpr bool stringp() const { return std::holds_alternative<String>(value); } + constexpr bool numberp() const { return std::holds_alternative<Number>(value); } + constexpr bool listp() const { return std::holds_alternative<List>(value); } + constexpr bool symbolp() const { return std::holds_alternative<Symbol>(value); } + constexpr bool symbolp(const Sexp::Symbol& sym) const {return symbolp() && symbol() == sym; } + constexpr bool nilp() const { return symbolp(nil_sym); } + + // Get the specific variant type. + const List& list() const { return std::get<List>(value); } + List& list() { return std::get<List>(value); } + const String& string() const { return std::get<String>(value); } + String& string() { return std::get<String>(value); } + const Number& number() const { return std::get<Number>(value); } + Number& number() { return std::get<Number>(value); } + const Symbol& symbol() const { return std::get<Symbol>(value); } + Symbol& symbol() { return std::get<Symbol>(value); } + + /** + * Constructors + */ + Sexp():value{List{}} {} // default: an empty list. + // Copy & move ctors + Sexp(const Sexp& other):value{other.value}{} + Sexp(Sexp&& other):value{std::move(other.value)}{} + // From various types + Sexp(const List& lst): value{lst} {} + Sexp(List&& lst): value{std::move(lst)} {} + Sexp(const String& str): value{str} {} + Sexp(String&& str): value{std::move(str)} {} + Sexp(const char *str): Sexp{std::string{str}} {} + Sexp(std::string_view sv): Sexp{std::string{sv}} {} + + template<typename N, typename = std::enable_if_t<std::is_integral_v<N>> > + Sexp(N n):value{static_cast<Number>(n)} {} + + Sexp(const Symbol& sym): value{sym} {} + Sexp(Symbol&& sym): value{std::move(sym)} {} + + template<typename S, typename T, typename... Args> + Sexp(S&& s, T&& t, Args&&... args): value{List()} { + auto& l{std::get<List>(value)}; + l.emplace_back(Sexp(std::forward<S>(s))); + l.emplace_back(Sexp(std::forward<T>(t))); + (l.emplace_back(Sexp(std::forward<Args>(args))), ...); + } + + /** + * Copy-assignment + * + * @param rhs another sexp + * + * @return the sexp + */ + Sexp& operator=(const Sexp& rhs) { + if (this != &rhs) + value = rhs.value; + return *this; + } + + /** + * Move-assignment + * + * @param rhs another sexp + * + * @return the sexp + */ + Sexp& operator=(Sexp&& rhs) { + if (this != &rhs) + value = std::move(rhs.value); + return *this; + } + + /** + * Get the type of value + * + * @return type + */ + constexpr Type type() const { return static_cast<Type>(value.index()); } + /** + * Get the name for some type + * + * @param t type + * + * @return name + */ + static constexpr std::string_view type_name(Type t) { + switch(t) { + case Type::String: + return "string"; + case Type::Number: + return "number"; + case Type::Symbol: + return "symbol"; + case Type::List: + return "list"; + default: + return "<error>"; + } + } + + /** + * Parse sexp from string + * + * @param str a string + * + * @return either an Sexp or an error + */ + static Result<Sexp> parse(const std::string& str); + + + /** + * List specific functionality + * + */ + using iterator = List::iterator; + using const_iterator = List::const_iterator; + + iterator begin() { return list().begin(); } + const_iterator begin() const { return list().begin(); } + const_iterator cbegin() const { return list().cbegin(); } + + iterator end() { return list().end(); } + const_iterator end() const { return list().end(); } + const_iterator cend() const { return list().cend(); } + + bool empty() const { return list().empty(); } + size_t size() const { return list().size(); } + void clear() { list().clear(); } + + /// Adding to lists + Sexp& add(const Sexp& s) { list().emplace_back(s); return *this; } + Sexp& add(Sexp&& s) { list().emplace_back(std::move(s)); return *this; } + Sexp& add() { return *this; } + + template <typename V1, typename V2, typename... Args> + Sexp& add(V1&& v1, V2&& v2, Args... args) { + return add(std::forward<V1>(v1)) + .add(std::forward<V2>(v2)) + .add(std::forward<Args>(args)...); + } + + /// Adding list elements + Sexp& add_list(Sexp&& l) { for (auto&& e: l) add(std::move(e)); return *this;}; + + /// Some convenience for the query parser + Sexp& front() { return list().front(); } + const Sexp& front() const { return list().front(); } + void pop_front() { list().erase(list().begin()); } + + Option<Sexp&> head() { if (listp()&&!empty()) return front(); else return Nothing; } + Option<const Sexp&> head() const { if (listp()&&!empty()) return front(); else return Nothing; } + + bool head_symbolp() const { + if (auto&& h{head()}; h) return h->symbolp(); else return false; + } + bool head_symbolp(const Symbol& sym) const { + if (head_symbolp()) return head()->symbolp(sym); else return false; + } + + /** + * Property lists (aka plists) + */ + + bool plistp() const { return listp() && plistp(cbegin(), cend()); } + Sexp& put_props() { return *this; } // Final case for template pack. + template <class PropType, class SexpType, typename... Args> + Sexp& put_props(PropType&& prop, SexpType&& sexp, Args... args) { + auto&& propname{std::string(prop)}; + return del_prop(propname) + .add(Symbol(std::move(propname)), + std::forward<SexpType>(sexp)) + .put_props(std::forward<Args>(args)...); + } + + /** + * Find the property value for some property by name + * + * @param p property name + * + * @return the property if found, or nothing + */ + const Option<const Sexp&> get_prop(const std::string& p) const { + if (auto&& it = find_prop(p, cbegin(), cend()); it != cend()) + return *(std::next(it)); + else + return Nothing; + } + /// Output to string + enum struct Format { + Default = 0, /**< Nothing in particular */ + SplitList = 1 << 0, /**< Insert newline after list item */ + TypeInfo = 1 << 1, /**< Show type-info */ + }; + + /** + * Get a string representation of the sexp + * + * @return str + */ + std::string to_string(Format fopts=Format::Default) const; + std::string to_json_string(Format fopts=Format::Default) const; + + Sexp& del_prop(const std::string& pname); + + /** + * Some useful constants + * + */ + static inline const auto nil_sym = Sexp::Symbol{"nil"}; + static inline const auto t_sym = Sexp::Symbol{"t"}; + +protected: + const_iterator find_prop(const std::string& s, const_iterator b, + const_iterator e) const; + bool plistp(const_iterator b, const_iterator e) const; +private: + iterator find_prop(const std::string& s,iterator b, + iterator e); + ValueType value; + + +}; + +MU_ENABLE_BITOPS(Sexp::Format); + +/** + * String-literal; allow for ":foo"_sym to be a symbol + */ +static inline Sexp::Symbol +operator"" _sym(const char* str, std::size_t n) +{ + return Sexp::Symbol{str}; +} + +static inline std::ostream& +operator<<(std::ostream& os, const Sexp::Type& stype) +{ + os << Sexp::type_name(stype); + return os; +} + + +static inline std::ostream& +operator<<(std::ostream& os, const Sexp& sexp) +{ + os << sexp.to_string(); + return os; +} + +} // namespace Mu + +#endif /* MU_SEXP_HH__ */ diff --git a/lib/utils/mu-test-utils.cc b/lib/utils/mu-test-utils.cc new file mode 100644 index 0000000..0d4a149 --- /dev/null +++ b/lib/utils/mu-test-utils.cc @@ -0,0 +1,140 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <glib.h> +#include <glib/gstdio.h> + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +#include <langinfo.h> +#include <locale.h> + +#include "utils/mu-utils.hh" +#include "utils/mu-test-utils.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-error.hh" + +using namespace Mu; + +/* LCOV_EXCL_START*/ +bool +Mu::mu_test_mu_hacker() +{ + return !!g_getenv("MU_HACKER"); +} +/* LCOV_EXCL_STOP*/ + + +const char* +Mu::set_tz(const char* tz) +{ + static const char* oldtz; + + oldtz = getenv("TZ"); + if (tz) + setenv("TZ", tz, 1); + else + unsetenv("TZ"); + + tzset(); + return oldtz; +} + +bool +Mu::set_en_us_utf8_locale() +{ + setenv("LC_ALL", "en_US.UTF-8", 1); + + if (auto str = setlocale(LC_ALL, "en_US.UTF-8"); !str) + return false; + + if (strcmp(nl_langinfo(CODESET), "UTF-8") != 0) + return false; + + return true; +} + +static void +black_hole(void) +{ + return; /* do nothing */ +} + +void +Mu::mu_test_init(int *argc, char ***argv) +{ + TempDir temp_dir; + + g_unsetenv("XAPIAN_CJK_NGRAM"); + g_setenv("MU_TEST", "yes", TRUE); + g_setenv("XDG_CACHE_HOME", temp_dir.path().c_str(), TRUE); + + setlocale(LC_ALL, ""); + + g_test_init(argc, argv, NULL); + + g_test_bug_base("https://github.com/djcb/mu/issues/"); + + if (!g_test_verbose()) + g_log_set_handler( + NULL, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | + G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION), + (GLogFunc)black_hole, NULL); +} + +void +Mu::allow_warnings() +{ + g_test_log_set_fatal_handler( + [](const char*, GLogLevelFlags, const char*, gpointer) { return FALSE; }, + {}); +} + +Mu::TempDir::TempDir(bool autodelete): autodelete_{autodelete} { + + if (auto res{make_temp_dir()}; !res) + throw res.error(); + else + path_ = std::move(*res); + + mu_debug("created '{}'", path_); +} + +Mu::TempDir::~TempDir() +{ + if (::access(path_.c_str(), F_OK) != 0) + return; /* nothing to do */ + + if (!autodelete_) { + mu_debug("_not_ deleting {}", path_); + return; + } + + if (auto&& res{run_command0({RM_PROGRAM, "-fr", path_})}; !res) { + /* LCOV_EXCL_START*/ + mu_warning("error removing {}: {}", path_, format_as(res.error())); + /* LCOV_EXCL_STOP*/ + } else + mu_debug("removed '{}'", path_); +} diff --git a/lib/utils/mu-test-utils.hh b/lib/utils/mu-test-utils.hh new file mode 100644 index 0000000..051230a --- /dev/null +++ b/lib/utils/mu-test-utils.hh @@ -0,0 +1,167 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_TEST_UTILS_HH__ +#define MU_TEST_UTILS_HH__ + +#include <initializer_list> +#include <string> +#include <utils/mu-utils.hh> +#include <utils/mu-result.hh> + +namespace Mu { + +/** + * mu wrapper for g_test_init. Sets environment variable MU_TEST to 1. + * + * @param argc + * @param argv + */ +void mu_test_init(int *argc, char ***argv); + + +/** + * Are we running in a MU_HACKER environment? + * + * @return true or false + */ +bool mu_test_mu_hacker(); + +/** + * set the timezone + * + * @param tz timezone + * + * @return the old timezone + */ +const char* set_tz(const char* tz); + +/** + * switch the locale to en_US.utf8, return TRUE if it succeeds + * + * @return true if the switch succeeds, false otherwise + */ +bool set_en_us_utf8_locale(); + + +/** + * For unit tests, assert two std::string's are equal. + * + * @param s1 string1 + * @param s2 string2 + */ +#define assert_equal(s1__,s2__) do { \ + std::string s1s__(s1__), s2s__(s2__); \ + g_assert_cmpstr(s1s__.c_str(), ==, s2s__.c_str()); \ + } while(0) + + +#define assert_equal_seq(seq1__, seq2__) do { \ + g_assert_cmpuint(seq1__.size(), ==, seq2__.size()); \ + size_t n__{}; \ + for (auto&& item__: seq1__) { \ + g_assert_true(item__ == seq2__.at(n__)); \ + ++n__; \ + } \ + } while(0) + +#define assert_equal_seq_str(seq1__, seq2__) do { \ + g_assert_cmpuint(seq1__.size(), ==, seq2__.size()); \ + size_t n__{}; \ + for (auto&& item__: seq1__) { \ + assert_equal(item__, seq2__.at(n__)); \ + ++n__; \ + } \ + } while(0) + +#define assert_valid_command(RCO) do { \ + assert_valid_result(RCO); \ + if ((RCO)->exit_code != 0 && !(RCO)->standard_err.empty()) \ + mu_printerrln("{}:{}: {}", \ + __FILE__, __LINE__, (RCO)->standard_err); \ + g_assert_cmpuint((RCO)->exit_code, ==, 0); \ +} while (0) + +/** + * For unit-tests, allow warnings in the current function. + * + */ +void allow_warnings(); + + +/** + * For unit-tests, a RAII tempdir. + * + */ +struct TempDir { + /** + * Construct a temporary directory + */ + TempDir(bool autodelete=true); + + /** + * DTOR; removes the temporary directory + * + * + * @return + */ + ~TempDir(); + + /** + * Path to the temporary directory + * + * @return the path. + */ + const std::string& path() const { return path_; } +private: + std::string path_; + const bool autodelete_; +}; + +static inline auto format_as(const TempDir& td) { + return td.path(); +} + + +/** + * Temporary (RAII) timezone + */ +struct TempTz { + TempTz(const char* tz) { + if (timezone_available(tz)) + old_tz_ = set_tz(tz); + else + old_tz_ = {}; + mu_debug("timezone '{}' {}available", tz, old_tz_ ? "": "not "); + } + ~TempTz() { + if (old_tz_) { + mu_debug("reset timezone to '{}'", old_tz_); + set_tz(old_tz_); + } + } + bool available() const { return !!old_tz_; } +private: + const char *old_tz_{}; +}; + +} // namepace Mu + + +#endif /* MU_TEST_UTILS_HH__ */ diff --git a/lib/utils/mu-unbroken.hh b/lib/utils/mu-unbroken.hh new file mode 100644 index 0000000..7c431d4 --- /dev/null +++ b/lib/utils/mu-unbroken.hh @@ -0,0 +1,127 @@ +// borrowed from Xapian; slightly adapted + +/* Copyright (c) 2007, 2008 Yung-chung Lin (henearkrxern@gmail.com) + * Copyright (c) 2011 Richard Boulton (richard@tartarus.org) + * Copyright (c) 2011 Brandon Schaefer (brandontschaefer@gmail.com) + * Copyright (c) 2011,2018,2019,2023 Olly Betts + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef MU_UNBROKEN_HH__ +#define MU_UNBROKEN_HH__ + +#include <algorithm> +#include <iterator> + +/** + * Does unichar p belong to a script without explicit word separators? + * + * @param p + * + * @return true or false + */ +static inline bool +is_unbroken_script(unsigned p) +{ + // Array containing the last value in each range of codepoints which + // are either all in scripts which are written without explicit word + // breaks, or all not in such scripts. + // + // We only include scripts here which ICU has dictionaries for. The + // same list is currently also used to decide which languages to do + // ngrams for, though perhaps that should use a separate list. + constexpr unsigned splits[] = { + // 0E00..0E7F; Thai, Lanna Tai, Pali + // 0E80..0EFF; Lao + 0x0E00 - 1, 0x0EFF, + // 1000..109F; Myanmar (Burmese) + 0x1000 - 1, 0x109F, + // 1100..11FF; Hangul Jamo + 0x1100 - 1, 0x11FF, + // 1780..17FF; Khmer + 0x1780 - 1, 0x17FF, + // 19E0..19FF; Khmer Symbols + 0x19E0 - 1, 0x19FF, + // 2E80..2EFF; CJK Radicals Supplement + // 2F00..2FDF; Kangxi Radicals + // 2FE0..2FFF; Ideographic Description Characters + // 3000..303F; CJK Symbols and Punctuation + // 3040..309F; Hiragana + // 30A0..30FF; Katakana + // 3100..312F; Bopomofo + // 3130..318F; Hangul Compatibility Jamo + // 3190..319F; Kanbun + // 31A0..31BF; Bopomofo Extended + // 31C0..31EF; CJK Strokes + // 31F0..31FF; Katakana Phonetic Extensions + // 3200..32FF; Enclosed CJK Letters and Months + // 3300..33FF; CJK Compatibility + // 3400..4DBF; CJK Unified Ideographs Extension A + // 4DC0..4DFF; Yijing Hexagram Symbols + // 4E00..9FFF; CJK Unified Ideographs + 0x2E80 - 1, 0x9FFF, + // A700..A71F; Modifier Tone Letters + 0xA700 - 1, 0xA71F, + // A960..A97F; Hangul Jamo Extended-A + 0xA960 - 1, 0xA97F, + // A9E0..A9FF; Myanmar Extended-B (Burmese) + 0xA9E0 - 1, 0xA9FF, + // AA60..AA7F; Myanmar Extended-A (Burmese) + 0xAA60 - 1, 0xAA7F, + // AC00..D7AF; Hangul Syllables + // D7B0..D7FF; Hangul Jamo Extended-B + 0xAC00 - 1, 0xD7FF, + // F900..FAFF; CJK Compatibility Ideographs + 0xF900 - 1, 0xFAFF, + // FE30..FE4F; CJK Compatibility Forms + 0xFE30 - 1, 0xFE4F, + // FF00..FFEF; Halfwidth and Fullwidth Forms + 0xFF00 - 1, 0xFFEF, + // 1AFF0..1AFFF; Kana Extended-B + // 1B000..1B0FF; Kana Supplement + // 1B100..1B12F; Kana Extended-A + // 1B130..1B16F; Small Kana Extension + 0x1AFF0 - 1, 0x1B16F, + // 1F200..1F2FF; Enclosed Ideographic Supplement + 0x1F200 - 1, 0x1F2FF, + // 20000..2A6DF; CJK Unified Ideographs Extension B + 0x20000 - 1, 0x2A6DF, + // 2A700..2B73F; CJK Unified Ideographs Extension C + // 2B740..2B81F; CJK Unified Ideographs Extension D + // 2B820..2CEAF; CJK Unified Ideographs Extension E + // 2CEB0..2EBEF; CJK Unified Ideographs Extension F + 0x2A700 - 1, 0x2EBEF, + // 2F800..2FA1F; CJK Compatibility Ideographs Supplement + 0x2F800 - 1, 0x2FA1F, + // 30000..3134F; CJK Unified Ideographs Extension G + // 31350..323AF; CJK Unified Ideographs Extension H + 0x30000 - 1, 0x323AF + }; + // Binary chop to find the first entry which is >= p. If it's an odd + // offset then the codepoint is in a script which needs splitting; if it's + // an even offset then it's not. + auto it = std::lower_bound(std::begin(splits), + std::end(splits), p); + + return ((it - splits) & 1); +} + + +#endif /* MU_UNBROKEN_HH__ */ diff --git a/lib/utils/mu-utils-file.cc b/lib/utils/mu-utils-file.cc new file mode 100644 index 0000000..3daea34 --- /dev/null +++ b/lib/utils/mu-utils-file.cc @@ -0,0 +1,521 @@ +/* +** Copyright (C) 2023-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include "mu-utils.hh" +#include "mu-utils-file.hh" + +#include <sys/stat.h> +#include <sys/wait.h> + +#include <glib.h> +#include <gio/gio.h> +#include <gio/gunixinputstream.h> + +#ifdef HAVE_WORDEXP_H +#include <wordexp.h> +#endif /*HAVE_WORDEXP_H*/ + +using namespace Mu; + + +bool +Mu::check_dir (const std::string& path, bool readable, bool writeable) +{ + const auto mode = F_OK | (readable ? R_OK : 0) | (writeable ? W_OK : 0); + + if (::access (path.c_str(), mode) != 0) + return false; + + struct stat statbuf{}; + if (::stat (path.c_str(), &statbuf) != 0) + return false; + + return S_ISDIR(statbuf.st_mode) ? true : false; +} + +uint8_t +Mu::determine_dtype (const std::string& path, bool use_lstat) +{ + int res; + struct stat statbuf{}; + + if (use_lstat) + res = ::lstat(path.c_str(), &statbuf); + else + res = ::stat(path.c_str(), &statbuf); + + if (res != 0) { + mu_warning ("{}stat failed on {}: {}", + use_lstat ? "l" : "", path, g_strerror(errno)); + return DT_UNKNOWN; + } + + /* we only care about dirs, regular files and links */ + if (S_ISREG (statbuf.st_mode)) + return DT_REG; + else if (S_ISDIR (statbuf.st_mode)) + return DT_DIR; + else if (S_ISLNK (statbuf.st_mode)) + return DT_LNK; + + return DT_UNKNOWN; +} + +std::string +Mu::canonicalize_filename(const std::string& path, const std::string& relative_to) +{ + auto str{to_string_opt_gchar( + g_canonicalize_filename( + path.c_str(), + relative_to.empty() ? nullptr : relative_to.c_str())).value()}; + + // remove trailing '/'... is this needed? + if (str[str.length()-1] == G_DIR_SEPARATOR) + str.erase(str.length() - 1); + + return str; +} + +std::string +Mu::basename(const std::string& path) +{ + return to_string_gchar(g_path_get_basename(path.c_str())); +} + +std::string +Mu::dirname(const std::string& path) +{ + return to_string_gchar(g_path_get_dirname(path.c_str())); +} + +Result<std::string> +Mu::make_temp_dir() +{ + GError *err{}; + if (auto tmpdir{g_dir_make_tmp("mu-tmp-XXXXXX", &err)}; !tmpdir) + return Err(Error::Code::File, &err, + "failed to create temporary directory"); + else + return Ok(to_string_gchar(std::move(tmpdir))); +} + + +Result<void> +Mu::remove_directory(const std::string& path) +{ + /* ugly */ + GError *err{}; + const auto cmd{mu_format("/bin/rm -rf '{}'", path)}; + if (!g_spawn_command_line_sync(cmd.c_str(), NULL, + NULL, NULL, &err)) + return Err(Error::Code::File, &err, "failed to remove {}", path); + else + return Ok(); +} + + + + +std::string +Mu::runtime_path(Mu::RuntimePath path, const std::string& muhome) +{ + auto [mu_cache, mu_config] = + std::invoke([&]()->std::pair<std::string, std::string> { + if (muhome.empty()) + return { join_paths(g_get_user_cache_dir(), "mu"), + join_paths(g_get_user_config_dir(), "mu")}; + else + return { muhome, muhome }; + }); + + switch (path) { + case Mu::RuntimePath::Cache: + return mu_cache; + case Mu::RuntimePath::XapianDb: + return join_paths(mu_cache, "xapian"); + case Mu::RuntimePath::LogFile: + return join_paths(mu_cache, "mu.log"); + case Mu::RuntimePath::Bookmarks: + return join_paths(mu_config, "bookmarks"); + case Mu::RuntimePath::Config: + return mu_config; + case Mu::RuntimePath::Scripts: + return join_paths(mu_config, "scripts"); + /*LCOV_EXCL_START*/ + default: + throw std::logic_error("unknown path"); + /*LCOV_EXCL_STOP*/ + } +} + +/* LCOV_EXCL_START*/ +static gpointer +cancel_wait(gpointer data) +{ + guint timeout, deadline; + GCancellable *cancel; + + cancel = (GCancellable*)data; + timeout = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(cancel), "timeout")); + deadline = g_get_monotonic_time() + 1000 * timeout; + + while (g_get_monotonic_time() < deadline && !g_cancellable_is_cancelled(cancel)) { + g_usleep(50 * 1000); /* 50 ms */ + g_thread_yield(); + } + + g_cancellable_cancel(cancel); + + return NULL; +} + +static void +cancel_wait_free(gpointer data) +{ + GThread *thread; + GCancellable *cancel; + + cancel = (GCancellable*)data; + thread = (GThread*)g_object_get_data(G_OBJECT(cancel), "thread"); + + g_cancellable_cancel(cancel); + g_thread_join(thread); +} + +GCancellable* +Mu::g_cancellable_new_with_timeout(guint timeout) +{ + GCancellable *cancel; + + cancel = g_cancellable_new(); + + g_object_set_data(G_OBJECT(cancel), "timeout", GUINT_TO_POINTER(timeout)); + g_object_set_data(G_OBJECT(cancel), "thread", + g_thread_new("cancel-wait", cancel_wait, cancel)); + g_object_set_data_full(G_OBJECT(cancel), "cancel", cancel, cancel_wait_free); + + return cancel; +} +/* LCOV_EXCL_STOP*/ + +/* LCOV_EXCL_START*/ +Result<std::string> +Mu::read_from_stdin() +{ + g_autoptr(GOutputStream) outmem = g_memory_output_stream_new_resizable(); + g_autoptr(GInputStream) input = g_unix_input_stream_new(STDIN_FILENO, TRUE); + //g_autoptr(GCancellable) cancel{maybe_cancellable_timeout(timeout)}; + + GError *err{}; + auto bytes = g_output_stream_splice(outmem, input, + static_cast<GOutputStreamSpliceFlags> + (G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | + G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET), + {}, &err); + + if (bytes < 0) + return Err(Error::Code::File, &err, "error reading from pipe"); + + return Ok(std::string{ + static_cast<const char*>(g_memory_output_stream_get_data( + G_MEMORY_OUTPUT_STREAM(outmem))), + g_memory_output_stream_get_size(G_MEMORY_OUTPUT_STREAM(outmem))}); +} +/* LCOV_EXCL_STOP*/ + + +/* + * Set the child to a group leader to avoid being killed when the + * parent group is killed. + */ +/*LCOV_EXCL_START*/ +static void +maybe_setsid (G_GNUC_UNUSED gpointer user_data) +{ +#if HAVE_SETSID + setsid(); +#endif /*HAVE_SETSID*/ +} +/*LCOV_EXCL_STOP*/ + +Result<Mu::CommandOutput> +Mu::run_command(std::initializer_list<std::string> args, bool try_setsid) +{ + std::vector<char*> argvec{}; + for (auto&& arg: args) + argvec.push_back(g_strdup(arg.c_str())); + argvec.push_back({}); + + { + std::vector<std::string> qargs{}; + for(auto&& arg: args) + qargs.emplace_back("'" + arg + "'"); + mu_debug("run-command: {}", fmt::join(qargs, " ")); + } + + GError *err{}; + int wait_status{}; + gchar *std_out{}, *std_err{}; + auto res = g_spawn_sync({}, + static_cast<char**>(argvec.data()), + {}, + (GSpawnFlags)(G_SPAWN_SEARCH_PATH), + try_setsid ? maybe_setsid : nullptr, {}, + &std_out, &std_err, &wait_status, &err); + + for (auto& a: argvec) + g_free(a); + + if (!res) + return Err(Error::Code::File, &err, "failed to execute command"); + else + return Ok(Mu::CommandOutput{ + WEXITSTATUS(wait_status), + to_string_gchar(std::move(std_out/*consumed*/)), + to_string_gchar(std::move(std_err/*consumed*/))}); +} + +Result<Mu::CommandOutput> +Mu::run_command0(std::initializer_list<std::string> args, bool try_setsid) +{ + if (auto&& res{run_command(args, try_setsid)}; !res) + return res; + else if (res->exit_code != 0) + return Err(Error::Code::File, "command returned {}: {}", + res->exit_code, + res->standard_err.empty() ? + std::string{"something went wrong"}: + res->standard_err); + else + return Ok(std::move(*res)); +} + + +Mu::Option<std::string> +Mu::program_in_path(const std::string& name) +{ + if (char *path = g_find_program_in_path(name.c_str()); path) + return to_string_gchar(std::move(path)/*consumes*/); + else + return Nothing; +} + + +/* LCOV_EXCL_START*/ +constexpr auto default_open_program = +#ifdef __APPLE__ + "open" +#else + "xdg-open" +#endif /*!__APPLE__*/ + ; + +Mu::Result<void> +Mu::play (const std::string& path) +{ + /* check nativity */ + GFile *gf = g_file_new_for_path(path.c_str()); + auto is_native = g_file_is_native(gf); + g_object_unref(gf); + if (!is_native) + return Err(Error::Code::File, "'{}' is not a native file", path); + + auto mpp{g_getenv ("MU_PLAY_PROGRAM")}; + const std::string prog{mpp ? mpp : default_open_program}; + + const auto program_path{program_in_path(prog)}; + if (!program_path) + return Err(Error::Code::File, "cannot find '{}' in path", prog); + else if (auto&& res{run_command({*program_path, path}, true/*try-setsid*/)}; !res) + return Err(std::move(res.error())); + else + return Ok(); +} +/* LCOV_EXCL_STOP*/ + + +Result<std::string> +expand_path_real(const std::string& str) +{ +#ifndef HAVE_WORDEXP_H + return Ok(std::string{str}); +#else + int res; + wordexp_t result{}; + + res = wordexp(str.c_str(), &result, 0); + if (res != 0) + return Err(Error::Code::File, "cannot expand {}; err={}", str, res); + else if (auto&n = result.we_wordc; n != 1) { + wordfree(&result); + return Err(Error::Code::File, "expected 1 expansions, but got {} for {}", n, str); + } + + std::string expanded{result.we_wordv[0]}; + wordfree(&result); + + return Ok(std::move(expanded)); + +#endif /*HAVE_WORDEXP_H*/ +} + + +Result<std::string> +Mu::expand_path(const std::string& str) +{ + if (auto&& res{expand_path_real(str)}; res) + return res; + + // failed... try quoting. + auto qstr{to_string_gchar(g_shell_quote(str.c_str()))}; + return expand_path_real(qstr); +} + + + +#ifdef BUILD_TESTS + +/* + * Tests. + * + */ + +#include <glib/gstdio.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include "utils/mu-test-utils.hh" + +static void +test_check_dir_01(void) +{ + if (g_access("/usr/bin", F_OK) == 0) { + g_assert_cmpuint( + check_dir("/usr/bin", true, false) == true, + ==, + g_access("/usr/bin", R_OK) == 0); + } +} + +static void +test_check_dir_02(void) +{ + if (g_access("/tmp", F_OK) == 0) { + g_assert_cmpuint( + check_dir("/tmp", false, true) == true, + ==, + g_access("/tmp", W_OK) == 0); + } +} + +static void +test_check_dir_03(void) +{ + if (g_access(".", F_OK) == 0) { + g_assert_cmpuint( + check_dir(".", true, true) == true, + ==, + g_access(".", W_OK | R_OK) == 0); + } +} + +static void +test_check_dir_04(void) +{ + /* not a dir, so it must be false */ + g_assert_cmpuint( + check_dir("test-util.c", true, true), + ==, + false); +} + +static void +test_determine_dtype_with_lstat(void) +{ + g_assert_cmpuint( + determine_dtype(MU_TESTMAILDIR, true), ==, DT_DIR); + g_assert_cmpuint( + determine_dtype(MU_TESTMAILDIR2, true), ==, DT_DIR); + g_assert_cmpuint( + determine_dtype(MU_TESTMAILDIR2 "/Foo/cur/mail5", true), + ==, DT_REG); +} + + +static void +test_program_in_path(void) +{ + g_assert_true(!!program_in_path("ls")); +} + +static void +test_join_paths() +{ + + assert_equal(join_paths(), ""); + assert_equal(join_paths("a"), "a"); + assert_equal(join_paths("a", "b"), "a/b"); + assert_equal(join_paths("/a/b///c/d//", "e"), "/a/b/c/d/e"); +} + +static void +test_runtime_paths() +{ + TempDir tdir; + + assert_equal(runtime_path(RuntimePath::Cache, tdir.path()), tdir.path()); + assert_equal(runtime_path(RuntimePath::XapianDb, tdir.path()), + join_paths(tdir.path(), "xapian")); + assert_equal(runtime_path(RuntimePath::Bookmarks, tdir.path()), + join_paths(tdir.path(), "bookmarks")); + assert_equal(runtime_path(RuntimePath::Config, tdir.path()), tdir.path()); + assert_equal(runtime_path(RuntimePath::Scripts, tdir.path()), + join_paths(tdir.path(), "scripts")); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + /* check_dir */ + g_test_add_func("/utils/check-dir-01", + test_check_dir_01); + g_test_add_func("/utils/check-dir-02", + test_check_dir_02); + g_test_add_func("/utils/check-dir-03", + test_check_dir_03); + g_test_add_func("/utils/check-dir-04", + test_check_dir_04); + g_test_add_func("/utils/determine-dtype-with-lstat", + test_determine_dtype_with_lstat); + g_test_add_func("/utils/program-in-path", + test_program_in_path); + g_test_add_func("/utils/join-paths", + test_join_paths); + g_test_add_func("/utils/runtime-paths", + test_runtime_paths); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-utils-file.hh b/lib/utils/mu-utils-file.hh new file mode 100644 index 0000000..7eb3ba5 --- /dev/null +++ b/lib/utils/mu-utils-file.hh @@ -0,0 +1,275 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_UTILS_FILE_HH__ +#define MU_UTILS_FILE_HH__ + +#include <string> +#include <cinttypes> +#include <sys/stat.h> + +#include <gio/gio.h> +#include <utils/mu-option.hh> +#include <utils/mu-result.hh> +#include <utils/mu-regex.hh> + +namespace Mu { + +/** + * Check if the directory has the given attributes + * + * @param path path to dir + * @param readable is it readable? false means "don't care" + * @param writeable is it writable? false means "don't care" + * + * @return true if is is a directory with given attributes; false otherwise. + */ +bool check_dir(const std::string& path, bool readable=false, bool writeable=false); + +/** + * See g_canonicalize_filename + * + * @param filename + * @param relative_to + * + * @return + */ +std::string canonicalize_filename(const std::string& path, const std::string& relative_to=""); + +/** + * Expand the filesystem path (as per wordexp(3)) + * + * @param str a filesystem path string + * + * @return the expanded string or some error + */ +Result<std::string> expand_path(const std::string& str); + + +/** + * Get the basename for path, i.e. without leading directory component, + * @see g_path_get_basename + * + * @param path + * + * @return the basename + */ +std::string basename(const std::string& path); + + +/** + * Get the dirname for path, i.e. without leading directory component, + * @see g_path_get_dirname + * + * @param path + * + * @return the dirname + */ +std::string dirname(const std::string& path); + + +/* + * for OSs without support for direntry->d_type, like Solaris + */ +#ifndef DT_UNKNOWN +enum { + DT_UNKNOWN = 0, +#define DT_UNKNOWN DT_UNKNOWN + DT_FIFO = 1, +#define DT_FIFO DT_FIFO + DT_CHR = 2, +#define DT_CHR DT_CHR + DT_DIR = 4, +#define DT_DIR DT_DIR + DT_BLK = 6, +#define DT_BLK DT_BLK + DT_REG = 8, +#define DT_REG DT_REG + DT_LNK = 10, +#define DT_LNK DT_LNK + DT_SOCK = 12, +#define DT_SOCK DT_SOCK + DT_WHT = 14 +#define DT_WHT DT_WHT +}; +#endif /*DT_UNKNOWN*/ + + /** + * get the d_type (as in direntry->d_type) for the file at path, using either + * stat(3) or lstat(3) + * + * @param path full path + * @param use_lstat whether to use lstat (otherwise use stat) + * + * @return DT_REG, DT_DIR, DT_LNK, or DT_UNKNOWN (other values are not supported + * currently) + */ +uint8_t determine_dtype(const std::string& path, bool use_lstat=false); + + +/** + * Well-known runtime paths + * + */ +enum struct RuntimePath { + XapianDb, + Cache, + LogFile, + Config, + Scripts, + Bookmarks +}; + +/** + * Get some well-known Path for internal use when don't have + * access to the command-line + * + * @param path the RuntimePath to find + * @param muhome path to muhome directory, or empty for the default. + * + * @return the path name + */ +std::string runtime_path(RuntimePath path, const std::string& muhome=""); + +/** + * Join path components into a path (with '/') + * + * @param s a string-convertible value + * @param args 0 or more string-convertible values + * + * @return the path + */ +static inline std::string join_paths() { return {}; } +template<typename S> std::string join_paths_(S&& s) { return std::string{s}; } +template<typename S, typename...Args> +std::string join_paths_(S&& s, Args...args) { + + static std::string sepa{"/"}; + auto&& str{std::string{std::forward<S>(s)}}; + if (auto&& rest{join_paths_(std::forward<Args>(args)...)}; !rest.empty()) + str += (sepa + rest); + return str; +} + +template<typename S, typename...Args> +std::string join_paths(S&& s, Args...args) { + + constexpr auto sepa = '/'; + auto path = join_paths_(std::forward<S>(s), std::forward<Args>(args)...); + + auto c{0U}; + while (c < path.size()) { + + if (path[c] != sepa) { + ++c; + continue; + } + + while (path[++c] == '/') { + path.erase(c, 1); + --c; + } + } + + return path; +} + + +/** + * Like g_cancellable_new(), but automatically cancels itself + * after timeout + * + * @param timeout timeout in millisecs + * + * @return A GCancellable* instances; free with g_object_unref() when + * no longer needed. + */ +GCancellable* g_cancellable_new_with_timeout(guint timeout); + +/** + * Read for standard input + * + * @return data from standard input or an error. + */ +Result<std::string> read_from_stdin(); + +/** + * Create a randomly-named temporary directory + * + * @return name of the temporary directory or an error. + */ +Result<std::string> make_temp_dir(); + + +/** + * Remove a directory, recursively. Does not have to be empty. + * + * @param path path to directory + * + * @return Ok() or an error. + */ +Result<void> remove_directory(const std::string& path); + +/** + * Run some system command. + * + * @param args a list of commmand line arguments (like argv) + * @param try_setsid whether to try setsid(2) (see its manpage for details) if this + * system supports it. + * + * @return Ok(exit code) or an error. Note that exit-code != 0 is _not_ + * considered an error from the perspective of run_command, but is for + * run_command0 + */ +struct CommandOutput { + int exit_code; + std::string standard_out; + std::string standard_err; +}; +Result<CommandOutput> run_command(std::initializer_list<std::string> args, + bool try_setsid=false); +Result<CommandOutput> run_command0(std::initializer_list<std::string> args, + bool try_setsid=false); + +/** + * Try to 'play' (ie., open with it's associated program) a file. On MacOS, the + * the program 'open' is used for this; on other platforms 'xdg-open' to do the + * actual opening. In addition you can set it to another program by setting thep + * MU_PLAY_PROGRAM environment variable + * + * This requires a 'native' file, see g_file_is_native() + * + * @param path full path of the file to open + * + * @return Ok() if succeeded, some error otherwise. + */ +Result<void> play(const std::string& path); + +/** + * Find program in PATH + * + * @param name the name of the program + * + * @return either the full path to program, or Nothing if not found. + */ +Option<std::string> program_in_path(const std::string& name); + +} // namespace Mu + +#endif /* MU_UTILS_FILE_HH__ */ diff --git a/lib/utils/mu-utils.cc b/lib/utils/mu-utils.cc new file mode 100644 index 0000000..6d36dc1 --- /dev/null +++ b/lib/utils/mu-utils.cc @@ -0,0 +1,713 @@ +/* +** Copyright (C) 2017-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This library is free software; you can redistribute it and/or +** modify it under the terms of the GNU Lesser General Public License +** as published by the Free Software Foundation; either version 2.1 +** of the License, or (at your option) any later version. +** +** This library is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** Lesser General Public License for more details. +** +** You should have received a copy of the GNU Lesser General Public +** License along with this library; if not, write to the Free +** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA +** 02110-1301, USA. +*/ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE +#include <stdexcept> +#endif /*_XOPEN_SOURCE*/ + +#include <array> + +#include <time.h> + +#define GNU_SOURCE +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> + +#include <string.h> +#include <iostream> +#include <algorithm> +#include <numeric> +#include <functional> +#include <cinttypes> +#include <charconv> +#include <limits> + +#include <glib.h> +#include <glib/gprintf.h> + +#include "mu-utils.hh" +#include "mu-unbroken.hh" + +#include "mu-error.hh" +#include "mu-option.hh" + +using namespace Mu; + +namespace { + +static gunichar +unichar_tolower(gunichar uc) +{ + if (!g_unichar_isalpha(uc)) + return uc; + + if (g_unichar_get_script(uc) != G_UNICODE_SCRIPT_LATIN) + return g_unichar_tolower(uc); + + switch (uc) { + case 0x00e6: + case 0x00c6: return 'e'; /* æ */ + case 0x00f8: return 'o'; /* ø */ + case 0x0110: + case 0x0111: + return 'd'; /* đ */ + /* todo: many more */ + default: return g_unichar_tolower(uc); + } +} + +/** + * gx_utf8_flatten: + * @str: a UTF-8 string + * @len: the length of @str, or -1 if it is %NULL-terminated + * + * Flatten some UTF-8 string; that is, downcase it and remove any diacritics. + * + * Returns: (transfer full): a flattened string, free with g_free(). + */ +static char* +gx_utf8_flatten(const gchar* str, gssize len) +{ + GString* gstr; + char * norm, *cur; + + g_return_val_if_fail(str, NULL); + + norm = g_utf8_normalize(str, len, G_NORMALIZE_ALL); + if (!norm) + return NULL; + + gstr = g_string_sized_new(strlen(norm)); + + for (cur = norm; cur && *cur; cur = g_utf8_next_char(cur)) { + gunichar uc; + + uc = g_utf8_get_char(cur); + if (g_unichar_combining_class(uc) != 0) + continue; + + g_string_append_unichar(gstr, unichar_tolower(uc)); + } + + g_free(norm); + + return g_string_free(gstr, FALSE); +} + +} // namespace + +bool +Mu::contains_unbroken_script(const char *str) +{ + while (str && *str) { + auto uc = g_utf8_get_char(str); + if (is_unbroken_script(uc)) + return true; + str = g_utf8_next_char(str); + } + + return false; +} + +std::string // gx_utf8_flatten +Mu::utf8_flatten(const char* str) +{ + if (!str) + return {}; + + if (contains_unbroken_script(str)) + return std::string{str}; + + // the pure-ascii case + if (g_str_is_ascii(str)) { + auto l = g_ascii_strdown(str, -1); + std::string s{l}; + g_free(l); + return s; + } + + // seems we need the big guns + char* flat = gx_utf8_flatten(str, -1); + if (!flat) + return {}; + + std::string s{flat}; + g_free(flat); + + return s; +} + + +/* turn \0-terminated buf into ascii (which is a utf8 subset); convert + * any non-ascii into '.' + */ +static char* +asciify_in_place (char *buf) +{ + char *c; + + g_return_val_if_fail (buf, NULL); + + for (c = buf; c && *c; ++c) { + if ((!isprint(*c) && !isspace (*c)) || !isascii(*c)) + *c = '.'; + } + + return buf; +} + +static char* +utf8ify (const char *buf) +{ + char *utf8; + + g_return_val_if_fail (buf, NULL); + + utf8 = g_strdup (buf); + + if (!g_utf8_validate (buf, -1, NULL)) + asciify_in_place (utf8); + + return utf8; +} + + +std::string +Mu::utf8_clean(const std::string& dirty) +{ + g_autoptr(GString) gstr = g_string_sized_new(dirty.length()); + g_autofree char *cstr = utf8ify(dirty.c_str()); + + for (auto cur = cstr; cur && *cur; cur = g_utf8_next_char(cur)) { + const gunichar uc = g_utf8_get_char(cur); + if (g_unichar_iscntrl(uc)) + g_string_append_c(gstr, ' '); + else + g_string_append_unichar(gstr, uc); + } + + return std::string{g_strstrip(gstr->str)}; +} + + +std::string +Mu::utf8_wordbreak(const std::string& txt) +{ + g_autoptr(GString) gstr = g_string_sized_new(txt.length()); + + bool spc{}; + for (auto cur = txt.c_str(); cur && *cur; cur = g_utf8_next_char(cur)) { + const gunichar uc = g_utf8_get_char(cur); + + if (g_unichar_iscntrl(uc)) { + g_string_append_c(gstr, ' '); + continue; + } + // inspired by Xapian's termgenerator. + + switch(uc) { + case '\'': + case '&': + case 0xb7: + case 0x5f4: + case 0x2019: + case 0x201b: + case 0x2027: + case ',': + case '.': + case ';': + case '+': + case '#': + case '-': + case 0x037e: // GREEK QUESTION MARK + case 0x0589: // ARMENIAN FULL STOP + case 0x060D: // ARABIC DATE SEPARATOR + case 0x07F8: // NKO COMMA + case 0x2044: // FRACTION SLASH + case 0xFE10: // PRESENTATION FORM FOR VERTICAL COMMA + case 0xFE13: // PRESENTATION FORM FOR VERTICAL COLON + case 0xFE14: // PRESENTATION FORM FOR VERTICAL SEMICOLON + if (spc) + break; + spc = true; + g_string_append_c(gstr, ' '); + break; + default: + spc = false; + g_string_append_unichar(gstr, uc); + break; + } + } + + return std::string{g_strstrip(gstr->str)}; +} + + +std::string +Mu::remove_ctrl(const std::string& str) +{ + char prev{'\0'}; + std::string result; + result.reserve(str.length()); + + for (auto&& c : str) { + if (::iscntrl(c) || c == ' ') { + if (prev != ' ') + result += prev = ' '; + } else + result += prev = c; + } + + return result; +} + +std::vector<std::string> +Mu::split(const std::string& str, const std::string& sepa) +{ + std::vector<std::string> vec; + size_t b = 0, e = 0; + + /* special cases */ + if (str.empty()) + return vec; + else if (sepa.empty()) { + for (auto&& c: str) + vec.emplace_back(1, c); + return vec; + } + + while (true) { + if (e = str.find(sepa, b); e != std::string::npos) { + vec.emplace_back(str.substr(b, e - b)); + b = e + sepa.length(); + } else { + vec.emplace_back(str.substr(b)); + break; + } + } + + return vec; +} + +std::vector<std::string> +Mu::split(const std::string& str, char sepa) +{ + std::vector<std::string> vec; + size_t b = 0, e = 0; + + /* special case */ + if (str.empty()) + return vec; + + while (true) { + if (e = str.find(sepa, b); e != std::string::npos) { + vec.emplace_back(str.substr(b, e - b)); + b = e + sizeof(sepa); + } else { + vec.emplace_back(str.substr(b)); + break; + } + } + + return vec; +} + +std::string +Mu::join(const std::vector<std::string>& svec, const std::string& sepa) +{ + if (svec.empty()) + return {}; + + + /* calculate the overall size beforehand, to avoid re-allocations. */ + size_t value_len = + std::accumulate(svec.cbegin(), svec.cend(), 0, + [](size_t size, const std::string& s) { + return size + s.size(); + }) + (svec.size() - 1) * sepa.length(); + + std::string value; + value.reserve(value_len); + + std::accumulate(svec.cbegin(), svec.cend(), std::ref(value), + [&](std::string& s1, const std::string& s2)->std::string& { + if (s1.empty()) + s1 = s2; + else { + s1.append(sepa); + s1.append(s2); + } + return s1; + }); + + return value; +} + +std::string +Mu::quote(const std::string& str) +{ + std::string res{"\""}; + + for (auto&& k : str) { + switch (k) { + case '"': res += "\\\""; break; + case '\\': res += "\\\\"; break; + default: res += k; + } + } + + return res + "\""; +} + +static Option<::time_t> +delta_ymwdhMs(const std::string& expr) +{ + char* endptr; + auto num = strtol(expr.c_str(), &endptr, 10); + if (num <= 0 || num > 9999 || !endptr || !*endptr) + return Nothing; + + int years, months, weeks, days, hours, minutes, seconds; + years = months = weeks = days = hours = minutes = seconds = 0; + + switch (endptr[0]) { + case 's': seconds = num; break; + case 'M': minutes = num; break; + case 'h': hours = num; break; + case 'd': days = num; break; + case 'w': weeks = num; break; + case 'm': months = num; break; + case 'y': years = num; break; + default: + return Nothing; + } + + GDateTime *then, *now = g_date_time_new_now_local(); + if (weeks != 0) + then = g_date_time_add_weeks(now, -weeks); + else + then = + g_date_time_add_full(now, -years, -months, -days, -hours, -minutes, -seconds); + + auto t = std::max<::time_t>(0, g_date_time_to_unix(then)); + + g_date_time_unref(then); + g_date_time_unref(now); + + return t; +} + +static Option<::time_t> +special_date_time(const std::string& d, bool is_first) +{ + if (d == "now") + return ::time({}); + + if (d == "today") { + GDateTime *dt, *midnight; + dt = g_date_time_new_now_local(); + + if (!is_first) { + GDateTime* tmp = dt; + dt = g_date_time_add_days(dt, 1); + g_date_time_unref(tmp); + } + + midnight = g_date_time_add_full(dt, + 0, + 0, + 0, + -g_date_time_get_hour(dt), + -g_date_time_get_minute(dt), + -g_date_time_get_second(dt)); + time_t t = MAX(0, (gint64)g_date_time_to_unix(midnight)); + g_date_time_unref(dt); + g_date_time_unref(midnight); + + return t; + } + + return Nothing; +} + +// if a date has a month day greater than the number of days in that month, +// change it to a valid date point to the last second in that month +static void +fixup_month(struct tm* tbuf) +{ + decltype(tbuf->tm_mday) max_days; + const auto month = tbuf->tm_mon + 1; + const auto year = tbuf->tm_year + 1900; + + switch (month) { + case 2: + if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) + max_days = 29; + else + max_days = 28; + break; + case 4: + case 6: + case 9: + case 11: + max_days = 30; + break; + default: + max_days = 31; + break; + } + + if (tbuf->tm_mday > max_days) { + tbuf->tm_mday = max_days; + tbuf->tm_hour = 23; + tbuf->tm_min = 59; + tbuf->tm_sec = 59; + } + } + + +Option<::time_t> +Mu::parse_date_time(const std::string& dstr, bool is_first, bool utc) +{ + struct tm tbuf{}; + GDateTime *dtime{}; + gint64 t; + + /* one-sided dates */ + if (dstr.empty()) + return is_first ? 0 : G_MAXINT64; + else if (dstr == "today" || dstr == "now") + return special_date_time(dstr, is_first); + else if (dstr.find_first_of("ymdwhMs") != std::string::npos) + return delta_ymwdhMs(dstr); + + constexpr char UserDateMin[] = "19700101000000"; + constexpr char UserDateMax[] = "29991231235959"; + + std::string date(is_first ? UserDateMin : UserDateMax); + std::copy_if(dstr.begin(), dstr.end(), date.begin(), [](auto c) { return isdigit(c); }); + + if (!::strptime(date.c_str(), "%Y%m%d%H%M%S", &tbuf) && + !::strptime(date.c_str(), "%Y%m%d%H%M", &tbuf) && + !::strptime(date.c_str(), "%Y%m%d%H", &tbuf) && + !::strptime(date.c_str(), "%Y%m%d", &tbuf) && + !::strptime(date.c_str(), "%Y%m", &tbuf) && + !::strptime(date.c_str(), "%Y", &tbuf)) + return Nothing; + + fixup_month(&tbuf); + dtime = utc ? + g_date_time_new_utc(tbuf.tm_year + 1900, + tbuf.tm_mon + 1, + tbuf.tm_mday, + tbuf.tm_hour, + tbuf.tm_min, + tbuf.tm_sec) : + g_date_time_new_local(tbuf.tm_year + 1900, + tbuf.tm_mon + 1, + tbuf.tm_mday, + tbuf.tm_hour, + tbuf.tm_min, + tbuf.tm_sec); + + t = g_date_time_to_unix(dtime); + g_date_time_unref(dtime); + + return to_time_t(t); +} + + +Option<int64_t> +Mu::parse_size(const std::string& val, bool is_first) +{ + int64_t size{-1}; + std::string str; + GRegex* rx; + GMatchInfo* minfo; + + /* one-sided ranges */ + if (val.empty()) + return is_first ? 0 : std::numeric_limits<int64_t>::max(); + + rx = g_regex_new("^(\\d+)(b|k|kb|m|mb|g|gb)?$", + G_REGEX_CASELESS, (GRegexMatchFlags)0, NULL); + minfo = NULL; + if (g_regex_match(rx, val.c_str(), (GRegexMatchFlags)0, &minfo)) { + + char* s; + s = g_match_info_fetch(minfo, 1); + size = atoll(s); + g_free(s); + + s = g_match_info_fetch(minfo, 2); + switch (s ? g_ascii_tolower(s[0]) : 0) { + case 'k': size *= 1024; break; + case 'm': size *= (1024 * 1024); break; + case 'g': size *= (1024 * 1024 * 1024); break; + default: break; + } + + g_free(s); + } + + g_regex_unref(rx); + g_match_info_unref(minfo); + + if (size < 0) + return Nothing; + else + return size; + +} + +std::string +Mu::to_lexnum(int64_t val) +{ + char buf[18]; /* 1 byte prefix + hex + \0 */ + buf[0] = 'f' + ::snprintf(buf + 1, sizeof(buf) - 1, "%" PRIx64, val); + return buf; +} + +int64_t +Mu::from_lexnum(const std::string& str) +{ + int64_t val{}; + std::from_chars(str.c_str() + 1, str.c_str() + str.size(), val, 16); + + return val; +} + +bool +Mu::locale_workaround() try +{ + // quite horrible... but some systems break otherwise with + // https://github.com/djcb/mu/issues/2252 + + try { + std::locale::global(std::locale("")); + } catch (const std::runtime_error& re) { + g_setenv("LC_ALL", "C", 1); + std::locale::global(std::locale("")); + } + + return true; + +} catch (...) { + return false; +} + +bool +Mu::timezone_available(const std::string& tz) +{ + const auto old_tz = g_getenv("TZ"); + + g_setenv("TZ", tz.c_str(), TRUE); + + auto tzone = g_time_zone_new_local (); + bool have_tz = g_strcmp0(g_time_zone_get_identifier(tzone), tz.c_str()) == 0; + g_time_zone_unref (tzone); + + if (old_tz) + g_setenv("TZ", old_tz, TRUE); + else + g_unsetenv("TZ"); + + return have_tz; +} + +std::string +Mu::summarize(const std::string& str, size_t max_lines) +{ + size_t nl_seen; + unsigned i,j; + gboolean last_was_blank; + + if (str.empty()) + return {}; + + /* len for summary <= original len */ + char *summary = g_new (gchar, str.length() + 1); + + /* copy the string up to max_lines lines, replace CR/LF/tab with + * single space */ + for (i = j = 0, nl_seen = 0, last_was_blank = TRUE; + nl_seen < max_lines && i < str.length(); ++i) { + + if (str[i] == '\n' || str[i] == '\r' || + str[i] == '\t' || str[i] == ' ' ) { + + if (str[i] == '\n') + ++nl_seen; + + /* no double-blanks or blank at end of str */ + if (!last_was_blank && str[i+1] != '\0') + summary[j++] = ' '; + + last_was_blank = TRUE; + } else { + + summary[j++] = str[i]; + last_was_blank = FALSE; + } + } + + summary[j] = '\0'; + + return to_string_gchar(std::move(summary)/*consumes*/); +} + + + + +static bool +locale_is_utf8 (void) +{ + const gchar *dummy; + static int is_utf8 = -1; + if (G_UNLIKELY(is_utf8 == -1)) + is_utf8 = g_get_charset(&dummy) ? 1 : 0; + + return !!is_utf8; +} + +bool +Mu::fputs_encoded (const std::string& str, FILE *stream) +{ + g_return_val_if_fail (stream, false); + + /* g_get_charset return TRUE when the locale is UTF8 */ + if (locale_is_utf8()) + return ::fputs (str.c_str(), stream) == EOF ? false: true; + + /* charset is _not_ utf8, so we need to convert it */ + char *conv{}; + if (g_utf8_validate (str.c_str(), -1, NULL)) + conv = g_locale_from_utf8 (str.c_str(), -1, {}, {}, {}); + + /* conversion failed; this happens because is some cases GMime may gives + * us non-UTF-8 strings from e.g. wrongly encoded message-subjects; if + * so, we escape the string */ + conv = conv ? conv : g_strescape (str.c_str(), "\n\t"); + int rv = conv ? ::fputs (conv, stream) : EOF; + g_free (conv); + + return (rv == EOF) ? false: true; +} diff --git a/lib/utils/mu-utils.hh b/lib/utils/mu-utils.hh new file mode 100644 index 0000000..783351f --- /dev/null +++ b/lib/utils/mu-utils.hh @@ -0,0 +1,640 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This library is free software; you can redistribute it and/or +** modify it under the terms of the GNU Lesser General Public License +** as published by the Free Software Foundation; either version 2.1 +** of the License, or (at your option) any later version. +** +** This library is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** Lesser General Public License for more details. +** +** You should have received a copy of the GNU Lesser General Public +** License along with this library; if not, write to the Free +** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA +** 02110-1301, USA. +*/ + +#ifndef MU_UTILS_HH__ +#define MU_UTILS_HH__ + +#include <string> +#include <string_view> +#include <sstream> +#include <vector> +#include <chrono> +#include <memory> +#include <cstdarg> +#include <glib.h> +#include <ostream> +#include <iostream> +#include <type_traits> +#include <algorithm> +#include <numeric> + +#include "mu-option.hh" + +#ifndef FMT_HEADER_ONLY +#define FMT_HEADER_ONLY +#endif /*FMT_HEADER_ONLY*/ +#include <fmt/format.h> +#include <fmt/core.h> +#include <fmt/chrono.h> +#include <fmt/ostream.h> + +namespace Mu { + +/* + * Separator characters used in various places; importantly, + * they are not used in UTF-8 + */ +constexpr const auto SepaChar1 = '\xfe'; +constexpr const auto SepaChar2 = '\xff'; + +/* + * Logging/printing/formatting functions connect libfmt with the Glib logging + * system. We wrap so perhaps at some point (C++23?) we can use std:: instead. + */ + +/* + * Debug/error/warning logging + * + * The 'noexcept' means that they _wilL_ terminate the program + * when the formatting fails (ie. a bug) + */ + +template<typename...T> +void mu_debug(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_DEBUG, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +template<typename...T> +void mu_info(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_INFO, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +template<typename...T> +void mu_message(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_MESSAGE, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +template<typename...T> +void mu_warning(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_WARNING, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +/* LCOV_EXCL_START*/ +template<typename...T> +void mu_critical(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_CRITICAL, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +template<typename...T> +void mu_error(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_ERROR, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +/* LCOV_EXCL_STOP*/ + +/* + * Printing; add our wrapper functions, one day we might be able to use std:: + */ + +template<typename...T> +void mu_print(fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::print(frm, std::forward<T>(args)...); +} +template<typename...T> +void mu_println(fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::println(frm, std::forward<T>(args)...); +} + +template<typename...T> +void mu_printerr(fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::print(stderr, frm, std::forward<T>(args)...); +} +template<typename...T> +void mu_printerrln(fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::println(stderr, frm, std::forward<T>(args)...); +} + + +/* stream */ +template<typename...T> +void mu_print(std::ostream& os, fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::print(os, frm, std::forward<T>(args)...); +} +template<typename...T> +void mu_println(std::ostream& os, fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::println(os, frm, std::forward<T>(args)...); +} + +/* + * Fprmatting + */ +template<typename...T> +std::string mu_format(fmt::format_string<T...> frm, T&&... args) noexcept { + return fmt::format(frm, std::forward<T>(args)...); +} + +template<typename Range> +auto mu_join(Range&& range, std::string_view sepa) { + return fmt::join(std::forward<Range>(range), sepa); +} + +template <typename T=::time_t> +std::tm mu_time(T t={}, bool use_utc=false) { + ::time_t tt{static_cast<::time_t>(t)}; + return use_utc ? fmt::gmtime(tt) : fmt::localtime(tt); +} + +using StringVec = std::vector<std::string>; + +/** + * Does the string contain script without explicit word separators? + * + * @param str a string + * + * @return true or false + */ +bool contains_unbroken_script(const char* str); +static inline bool contains_unbroken_script(const std::string& str) { + return contains_unbroken_script(str.c_str()); +} + +/** + * Flatten a string -- down-case and fold diacritics. + * + * @param str a string + * + * @return a flattened string + */ +std::string utf8_flatten(const char* str); +inline std::string +utf8_flatten(const std::string& s) { + return utf8_flatten(s.c_str()); +} + +/** + * Replace all control characters with spaces, and remove leading and trailing space. + * + * @param dirty an unclean string + * + * @return a cleaned-up string. + */ +std::string utf8_clean(const std::string& dirty); + + +/** + * Replace all wordbreak chars (as recognized by Xapian by single SPC) + * + * @param txt text + * + * @return string + */ +std::string utf8_wordbreak(const std::string& txt); + + +/** + * Remove ctrl characters, replacing them with ' '; subsequent + * ctrl characters are replaced by a single ' ' + * + * @param str a string + * + * @return the string without control characters + */ +std::string remove_ctrl(const std::string& str); + +/** + * Split a string in parts. As a special case, splitting an empty string + * yields an empty vector (not a vector with a single empty element) + * + * @param str a string + * @param sepa the separator + * + * @return the parts. + */ +std::vector<std::string> split(const std::string& str, const std::string& sepa); + +/** + * Split a string in parts. As a special case, splitting an empty string + * yields an empty vector (not a vector with a single empty element) + * + * @param str a string + * @param sepa the separator + * + * @return the parts. + */ +std::vector<std::string> split(const std::string& str, char sepa); + +/** + * Join the strings in svec into a string, separated by sepa + * + * @param svec a string vector + * @param sepa separator + * + * @return string + */ +std::string join(const std::vector<std::string>& svec, const std::string& sepa); +static inline std::string join(const std::vector<std::string>& svec, char sepa) { + return join(svec, std::string(1, sepa)); +} + +/** + * write a string (assumed to be in utf8-format) to a stream, + * converted to the current locale + * + * @param str a string + * @param stream a stream + * + * @return true if printing worked, false otherwise + */ +bool fputs_encoded (const std::string& str, FILE *stream); + +/** + * print a fmt-style formatted string (assumed to be in utf8-format) to stdout, + * converted to the current locale + * + * @param a standard fmt-style format string, followed by a parameter list + * + * @return true if printing worked, false otherwise + */ +template<typename...T> +static inline bool mu_print_encoded(fmt::format_string<T...> frm, T&&... args) noexcept { + return fputs_encoded(fmt::format(frm, std::forward<T>(args)...), + stdout); +} + +/** + * Convert an int64_t to a time_t, clamping it within the range. + * + * This is only doing anything when using a 32-bit time_t value. This doesn't + * solve the 3038 problem, but at least allows for clearly marking where we + * convert + * + * @param t some 64-bit value that encodes a Unix time. + * + * @return a time_t value + */ +constexpr ::time_t time_t_min = 0; +constexpr ::time_t time_t_max = std::numeric_limits<::time_t>::max(); +constexpr ::time_t to_time_t(int64_t t) { + return std::clamp(t, + static_cast<int64_t>(time_t_min), + static_cast<int64_t>(time_t_max)); +} + + +/** + * Parse a date string to the corresponding time_t + * * + * @param date the date expressed a YYYYMMDDHHMMSS or any n... of the first + * characters, using the local timezone. Non-digits are ignored, + * so 2018-05-05 is equivalent to 20180505. + * @param first whether to fill out incomplete dates to the start (@true) or the + * end (@false); ie. either 1972 -> 197201010000 or 1972 -> 197212312359 + * @param use_utc interpret @param date as UTC + * + * @return the corresponding time_t or Nothing if parsing failed. + */ +Option<::time_t> parse_date_time(const std::string& date, bool first, bool use_utc=false); + +/** + * Crudely convert HTML to plain text. This attempts to scrape the + * human-readable text from html-email so we can use it for indexing. + * + * @param html html + * + * @return plain text + */ +std::string html_to_text(const std::string& html); + +/** + * Hack to avoid locale crashes + * + * @return true if setting locale worked; false otherwise + */ +bool locale_workaround(); + + +/** + * Is the given timezone available? For tests + * + * @param tz a timezone, such as Europe/Helsinki + * + * @return true or false + */ +bool timezone_available(const std::string& tz); + + +// https://stackoverflow.com/questions/19053351/how-do-i-use-a-custom-deleter-with-a-stdunique-ptr-member +template <auto fn> +struct deleter_from_fn { + template <typename T> + constexpr void operator()(T* arg) const { + fn(arg); + } +}; +template <typename T, auto fn> +using deletable_unique_ptr = std::unique_ptr<T, deleter_from_fn<fn>>; + + + +using Clock = std::chrono::steady_clock; +using Duration = Clock::duration; + +template <typename Unit> +constexpr int64_t +to_unit(Duration d) +{ + using namespace std::chrono; + return duration_cast<Unit>(d).count(); +} + +constexpr int64_t +to_s(Duration d) +{ + return to_unit<std::chrono::seconds>(d); +} +constexpr int64_t +to_ms(Duration d) +{ + return to_unit<std::chrono::milliseconds>(d); +} +constexpr int64_t +to_us(Duration d) +{ + return to_unit<std::chrono::microseconds>(d); +} + +struct StopWatch { + using Clock = std::chrono::steady_clock; + StopWatch(const std::string name) : start_{Clock::now()}, name_{name} {} + ~StopWatch() { + const auto us{static_cast<double>(to_us(Clock::now() - start_))}; + /* LCOV_EXCL_START*/ + if (us > 2000000) + mu_debug("sw: {}: finished after {:.1f} s", name_, us / 1000000); + /* LCOV_EXCL_STOP*/ + else if (us > 2000) + mu_debug("sw: {}: finished after {:.1f} ms", name_, us / 1000); + else + mu_debug("sw: {}: finished after {} us", name_, us); + } +private: + Clock::time_point start_; + std::string name_; +}; + +/** + * Convert a size string to a size in bytes + * + * @param sizestr the size string + * @param first + * + * @return the size or Nothing if parsing failed + */ +Option<int64_t> parse_size(const std::string& sizestr, bool first); + +/** + * Convert a size into a size in bytes string + * + * @param size the size + * @param first + * + * @return the size expressed as a string with the decimal number of bytes + */ +std::string size_to_string(int64_t size); + +/** + * get a crude 'summary' of the string, ie. the first /n/ lines of the strings, + * with all newlines removed, replaced by single spaces + * + * @param str the source string + * @param max_lines the maximum number of lines to include in the summary + * + * @return a newly allocated string with the summary. use g_free to free it. + */ +std::string summarize(const std::string& str, size_t max_lines); + + +/** + * Quote & escape a string for " and \ + * + * @param str a string + * + * @return quoted string + */ +std::string quote(const std::string& str); + + +/** + * Convert any ostreamable<< value to a string + * + * @param t the value + * + * @return a std::string + */ +template <typename T> +static inline std::string +to_string(const T& val) +{ + std::stringstream sstr; + sstr << val; + + return sstr.str(); +} +/** + * Convert to std::string to a std::string_view + * Careful with the lifetimes! + * + * @param s a string + * + * @return a string_view + */ +static inline std::string_view +to_string_view(const std::string& s) +{ + return std::string_view{s.data(), s.size()}; +} + +/** + * Consume a gchar and return a std::string + * + * @param str a gchar* (consumed/freed) + * + * @return a std::string, empty if gchar was {} + */ +static inline std::string +to_string_gchar(gchar*&& str) +{ + std::string s(str?str:""); + g_free(str); + return s; +} + + +/* + * Lexnums are lexicographically sortable string representations of non-negative + * integers. Start with 'f' + length of hex-representation number, followed by + * the hex representation itself. So, + * + * 0 -> 'g0' + * 1 -> 'g1' + * 10 -> 'ga' + * 16 -> 'h10' + * + * etc. + */ +std::string to_lexnum(int64_t val); +int64_t from_lexnum(const std::string& str); + +/** + * Like std::find_if, but using sequence instead of a range. + * + * @param seq some std::find_if compatible sequence + * @param pred a predicate + * + * @return an iterator + */ +template<typename Sequence, typename UnaryPredicate> +typename Sequence::const_iterator seq_find_if(const Sequence& seq, UnaryPredicate pred) { + return std::find_if(seq.cbegin(), seq.cend(), pred); +} + +/** + * Is pred(element) true for at least one element of sequence? + * + * @param seq sequence + * @param pred a predicate + * + * @return true or false + */ +template<typename Sequence, typename UnaryPredicate> +bool seq_some(const Sequence& seq, UnaryPredicate pred) { + return seq_find_if(seq, pred) != seq.cend(); +} + +/** + * Create a sequence that has all element of seq for which pred is true + * + * @param seq sequence + * @param pred false + * + * @return sequence + */ +template<typename Sequence, typename UnaryPredicate> +Sequence seq_filter(const Sequence& seq, UnaryPredicate pred) { + Sequence res; + std::copy_if(seq.begin(), seq.end(), std::back_inserter(res), pred); + return res; +} + +/** + * Create a sequence that has all element of seq for which pred is false + * + * @param seq sequence + * @param pred false + * + * @return sequence + */ +template<typename Sequence, typename UnaryPredicate> +Sequence seq_remove(const Sequence& seq, UnaryPredicate pred) { + Sequence res; + std::remove_copy_if(seq.begin(), seq.end(), std::back_inserter(res), pred); + return res; +} + +template<typename Sequence, typename Compare> +void seq_sort(Sequence& seq, Compare cmp) { std::sort(seq.begin(), seq.end(), cmp); } + + +/** + * Like std::accumulate, but using a sequence instead of a range. + * + * @param seq some std::accumulate compatible sequence + * @param init the initial value + * @param op binary operation to calculate the next element + * + * @return the result value. + */ +template<typename Sequence, typename ResultType, typename BinaryOp> +ResultType seq_fold(const Sequence& seq, ResultType init, BinaryOp op) { + return std::accumulate(seq.cbegin(), seq.cend(), init, op); +} + +template<typename Sequence, typename UnaryOp> +void seq_for_each(const Sequence& seq, UnaryOp op) { + std::for_each(seq.cbegin(), seq.cend(), op); +} + +struct MaybeAnsi { + explicit MaybeAnsi(bool use_color) : color_{use_color} {} + + enum struct Color { + Black = 30, + Red = 31, + Green = 32, + Yellow = 33, + Blue = 34, + Magenta = 35, + Cyan = 36, + White = 37, + + BrightBlack = 90, + BrightRed = 91, + BrightGreen = 92, + BrightYellow = 93, + BrightBlue = 94, + BrightMagenta = 95, + BrightCyan = 96, + BrightWhite = 97, + }; + + std::string fg(Color c) const { return ansi(c, true); } + std::string bg(Color c) const { return ansi(c, false); } + + std::string reset() const { return color_ ? "\x1b[0m" : ""; } + +private: + std::string ansi(Color c, bool fg = true) const + { + return color_ ? mu_format("\x1b[{}m", + static_cast<int>(c) + (fg ? 0 : 10)) : ""; + } + + const bool color_; +}; + +#define MU_COLOR_RED "\x1b[31m" +#define MU_COLOR_GREEN "\x1b[32m" +#define MU_COLOR_YELLOW "\x1b[33m" +#define MU_COLOR_BLUE "\x1b[34m" +#define MU_COLOR_MAGENTA "\x1b[35m" +#define MU_COLOR_CYAN "\x1b[36m" +#define MU_COLOR_DEFAULT "\x1b[0m" + + +/// Allow using enum structs as bitflags +#define MU_TO_NUM(ET, ELM) std::underlying_type_t<ET>(ELM) +#define MU_TO_ENUM(ET, NUM) static_cast<ET>(NUM) +#define MU_ENABLE_BITOPS(ET) \ + constexpr ET operator&(ET e1, ET e2) { \ + return MU_TO_ENUM(ET, MU_TO_NUM(ET, e1) & MU_TO_NUM(ET, e2)); \ + } \ + constexpr ET operator|(ET e1, ET e2) { \ + return MU_TO_ENUM(ET, MU_TO_NUM(ET, e1) | MU_TO_NUM(ET, e2)); \ + } \ + constexpr ET operator~(ET e) { return MU_TO_ENUM(ET, ~(MU_TO_NUM(ET, e))); } \ + constexpr bool any_of(ET e) { return MU_TO_NUM(ET, e) != 0; } \ + constexpr bool none_of(ET e) { return MU_TO_NUM(ET, e) == 0; } \ + constexpr bool one_of(ET e1, ET e2) { return (e1 & e2) == e2; } \ + constexpr ET& operator&=(ET& e1, ET e2) { return e1 = e1 & e2; } \ + constexpr ET& operator|=(ET& e1, ET e2) { return e1 = e1 | e2; } \ + static_assert(1==1) // require a semicolon + +} // namespace Mu + +#endif /* MU_UTILS_HH__ */ diff --git a/lib/utils/tests/meson.build b/lib/utils/tests/meson.build new file mode 100644 index 0000000..9c2883b --- /dev/null +++ b/lib/utils/tests/meson.build @@ -0,0 +1,83 @@ +## Copyright (C) 2021-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +################################################################################ +# tests + + +# +# tests +# +test('test-sexp', + executable('test-sexp', '../mu-sexp.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-regex', + executable('test-regex', '../mu-regex.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-command-handler', + executable('test-command-handler', '../mu-command-handler.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-utils-file', + executable('test-utils-file', '../mu-utils-file.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gio_unix_dep,config_h_dep, lib_mu_utils_dep])) + +test('test-logger', + executable('test-logger', '../mu-logger.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep, thread_dep ])) + +test('test-option', + executable('test-option', '../mu-option.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep ])) + +test('test-lang-detector', + executable('test-lang-detector', '../mu-lang-detector.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [ config_h_dep, glib_dep, lib_mu_utils_dep ])) + +test('test-html-to-text', + executable('test-html-to-text', '../mu-html-to-text.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-error', + executable('test-error', '../mu-error.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-mu-utils', + executable('test-mu-utils', + 'test-utils.cc', + install: false, + dependencies: [glib_dep, lib_mu_utils_dep])) diff --git a/lib/utils/tests/test-utils.cc b/lib/utils/tests/test-utils.cc new file mode 100644 index 0000000..fe0d075 --- /dev/null +++ b/lib/utils/tests/test-utils.cc @@ -0,0 +1,343 @@ +/* +** Copyright (C) 2017-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This library is free software; you can redistribute it and/or +** modify it under the terms of the GNU Lesser General Public License +** as published by the Free Software Foundation; either version 2.1 +** of the License, or (at your option) any later version. +** +** This library is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** Lesser General Public License for more details. +** +** You should have received a copy of the GNU Lesser General Public +** License along with this library; if not, write to the Free +** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA +** 02110-1301, USA. +*/ + +#include <vector> +#include <glib.h> + +#include <iostream> +#include <sstream> +#include <functional> +#include <array> + +#include "mu-utils.hh" +#include "mu-test-utils.hh" +#include "mu-error.hh" + +using namespace Mu; + + +struct Case { + const std::string expr; + bool is_first{}; + const std::string expected; +}; +using CaseVec = std::vector<Case>; +using ProcFunc = std::function<std::string(std::string, bool)>; + +static void +test_cases(const CaseVec& cases, ProcFunc proc) +{ + for (const auto& casus : cases) { + const auto res = proc(casus.expr, casus.is_first); + //mu_println("'{}'\n'{}'", casus.expected, res); + assert_equal(casus.expected, res); + } +} + +static void +test_date_basic() +{ + const auto hki = "Europe/Helsinki"; + + // ensure we have the needed TZ or skip the test. + if (!timezone_available(hki)) { + g_test_skip("timezone Europe/Helsinki not available"); + return; + } + + g_setenv("TZ", hki, TRUE); + std::vector<std::tuple<const char*, bool/*is_first*/, ::time_t>> cases = {{ + {"2015-09-18T09:10:23", true, 1442556623}, + {"1972-12-14T09:10:23", true, 93165023}, + {"1972-12-14T09:10", true, 93165000}, + {"1854-11-18T17:10:23", true, 0}, + + {"2000-02-31T09:10:23", true, 951861599}, + {"2000-02-29T23:59:59", true, 951861599}, + + {"20220602", true, 1654117200}, + {"20220605", false, 1654462799}, + + {"202206", true, 1654030800}, + {"202206", false, 1656622799}, + + {"2016", true, 1451599200}, + {"2016", false, 1483221599}, + + // {"fnorb", true, -1}, + // {"fnorb", false, -1}, + {"", false, time_t_max}, + {"", true, time_t_min} + }}; + + for (auto& test: cases) { + if (g_test_verbose()) + g_debug("checking %s", std::get<0>(test)); + g_assert_cmpuint(parse_date_time(std::get<0>(test), + std::get<1>(test)).value_or(-1),==, + std::get<2>(test)); + } +} + +static void +test_date_ymwdhMs(void) +{ + struct testcase { + std::string expr; + int64_t diff; + int tolerance; + }; + + std::array<testcase, 7> cases = {{ + {"7s", 7, 1}, + {"3M", 3 * 60, 1}, + {"3h", 3 * 60 * 60, 1}, + {"21d", 21 * 24 * 60 * 60, 3600 + 1}, + {"2w", 2 * 7 * 24 * 60 * 60, 3600 + 1}, + {"2y", 2 * 365 * 24 * 60 * 60, 24 * 3600 + 1}, + {"3m", 3 * 30 * 24 * 60 * 60, 3 * 24 * 3600 + 1} + }}; + + for (auto&& tcase: cases) { + const auto date = parse_date_time(tcase.expr, true); + g_assert_true(date); + const auto diff = ::time({}) - *date; + if (g_test_verbose()) + std::cerr << tcase.expr << ' ' << diff << ' ' << tcase.diff << '\n'; + + g_assert_true(tcase.diff - diff <= tcase.tolerance); + } + + // note: perhaps it'd be nice if we'd detect this error; + // currently we're being rather tolerant + // g_assert_false(!!parse_date_time("25q", false)); +} + +static void +test_parse_size() +{ + constexpr std::array<std::tuple<const char*, bool, int64_t>, 6> cases = {{ + { "456", false, 456 }, + { "", false, G_MAXINT64 }, + { "", true, 0 }, + { "2K", false, 2048 }, + { "2M", true, 2097152 }, + { "5G", true, 5368709120 } + }}; + for(auto&& test: cases) { + g_assert_cmpint(parse_size(std::get<0>(test), std::get<1>(test)) + .value_or(-1), ==, std::get<2>(test)); + } + + g_assert_false(!!parse_size("-1", true)); + g_assert_false(!!parse_size("scoobydoobydoo", false)); +} + +static void +test_flatten() +{ + CaseVec cases = { + {"Менделе́ев", true, "менделеев"}, + {"", true, ""}, + {"Ångström", true, "angstrom"}, + {"đodø", true, "dodo"}, + + // don't touch combining characters in CJK etc. + {"スポンサーシップ募集",true, "スポンサーシップ募集"} + }; + + test_cases(cases, [](auto s, auto f) { return utf8_flatten(s); }); +} + +static void +test_remove_ctrl() +{ + CaseVec cases = { + {"Foo\n\nbar", true, "Foo bar"}, + {"", false, ""}, + {" ", false, " "}, + {"Hello World ", false, "Hello World "}, + {"Ångström", false, "Ångström"}, + }; + + test_cases(cases, [](auto s, auto f) { return remove_ctrl(s); }); +} + +static void +test_clean() +{ + CaseVec cases = { + {"\t a\t\nb ", true, "a b"}, + {"", true, ""}, + {"Ångström", true, "Ångström"}, + {"\345\245", true, ".."}, + }; + + test_cases(cases, [](auto s, auto f) { return utf8_clean(s); }); +} + + +static void +test_word_break() +{ + CaseVec cases = { + {"aap+noot&mies", true, "aap noot mies"}, + {"hallo", true, "hallo"}, + {" foo-bar###cuux,fnorb ", true, "foo bar cuux fnorb"}, + {"eyes\nof\tMedusa", true, "eyes of Medusa"}, + }; + + test_cases(cases, [](auto s, auto f) { return utf8_wordbreak(s); }); +} + + +static void +test_format() +{ + g_assert_true(mu_format("hello {}", "world") == "hello world"); + g_assert_true(mu_format("hello {}, {}", "world", 123) == "hello world, 123"); +} + +static void +test_split() +{ + using svec = std::vector<std::string>; + auto assert_equal_svec=[](const svec& sv1, const svec& sv2) { + g_assert_cmpuint(sv1.size(),==,sv2.size()); + for (auto i = 0U; i != sv1.size(); ++i) + g_assert_cmpstr(sv1[i].c_str(),==,sv2[i].c_str()); + }; + + // string sepa + assert_equal_svec(split("axbxc", "x"), {"a", "b", "c"}); + assert_equal_svec(split("axbxcx", "x"), {"a", "b", "c", ""}); + assert_equal_svec(split("", "boo"), {}); + assert_equal_svec(split("ayybyyc", "yy"), {"a", "b", "c"}); + assert_equal_svec(split("abc", ""), {"a", "b", "c"}); + assert_equal_svec(split("", "boo"), {}); + + // char sepa + assert_equal_svec(split("axbxc", 'x'), {"a", "b", "c"}); + assert_equal_svec(split("axbxcx", 'x'), {"a", "b", "c", ""}); +} + +static void +test_join() +{ + assert_equal(join({"a", "b", "c"}, "x"), "axbxc"); + assert_equal(join({"a", "b", "c"}, ""), "abc"); + assert_equal(join({},"foo"), ""); + assert_equal(join({"d", "e", "f"}, "foo"), "dfooefoof"); +} + + +enum struct Bits { None = 0, Bit1 = 1 << 0, Bit2 = 1 << 1 }; +MU_ENABLE_BITOPS(Bits); + +static void +test_define_bitmap() +{ + g_assert_cmpuint((guint)Bits::None, ==, (guint)0); + g_assert_cmpuint((guint)Bits::Bit1, ==, (guint)1); + g_assert_cmpuint((guint)Bits::Bit2, ==, (guint)2); + + g_assert_cmpuint((guint)(Bits::Bit1 | Bits::Bit2), ==, (guint)3); + g_assert_cmpuint((guint)(Bits::Bit1 & Bits::Bit2), ==, (guint)0); + + g_assert_cmpuint((guint)(Bits::Bit1 & (~Bits::Bit2)), ==, (guint)1); + + { + Bits b{Bits::Bit1}; + b |= Bits::Bit2; + g_assert_cmpuint((guint)b, ==, (guint)3); + } + + { + Bits b{Bits::Bit1}; + b &= Bits::Bit1; + g_assert_cmpuint((guint)b, ==, (guint)1); + } +} + +static void +test_to_from_lexnum() +{ + assert_equal(to_lexnum(0), "g0"); + assert_equal(to_lexnum(100), "h64"); + assert_equal(to_lexnum(12345), "j3039"); + + g_assert_cmpuint(from_lexnum(to_lexnum(0)), ==, 0); + g_assert_cmpuint(from_lexnum(to_lexnum(7777)), ==, 7777); + g_assert_cmpuint(from_lexnum(to_lexnum(9876543)), ==, 9876543); +} + +static void +test_locale_workaround() +{ + g_assert_true(locale_workaround()); + + g_setenv("LC_ALL", "BOO", 1); + + g_assert_true(locale_workaround()); +} + + +static void +test_summarize(void) +{ + const char *txt = + "Khiron was fortified and made the seat of a pargana during " + "the reign of Asaf-ud-Daula.\n\the headquarters had previously " + "been at Satanpur since its foundation and fortification by " + "the Bais raja Sathna.\n\nKhiron was also historically the seat " + "of a taluqdari estate belonging to a Janwar dynasty.\n" + "There were also several Kayasth qanungo families, " + "including many descended from Rai Sahib Rai, who had been " + "a chakladar under the Nawabs of Awadh."; + + const auto summ = summarize(txt, 3); + g_assert_cmpstr(summ.c_str(), ==, + "Khiron was fortified and made the seat of a pargana " + "during the reign of Asaf-ud-Daula. he headquarters had " + "previously been at Satanpur since its foundation and " + "fortification by the Bais raja Sathna. "); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/utils/date-basic", test_date_basic); + g_test_add_func("/utils/date-ymwdhMs", test_date_ymwdhMs); + g_test_add_func("/utils/parse-size", test_parse_size); + g_test_add_func("/utils/flatten", test_flatten); + g_test_add_func("/utils/remove-ctrl", test_remove_ctrl); + g_test_add_func("/utils/clean", test_clean); + g_test_add_func("/utils/word-break", test_word_break); + g_test_add_func("/utils/format", test_format); + g_test_add_func("/utils/summarize", test_summarize); + g_test_add_func("/utils/split", test_split); + g_test_add_func("/utils/join", test_join); + g_test_add_func("/utils/define-bitmap", test_define_bitmap); + g_test_add_func("/utils/to-from-lexnum", test_to_from_lexnum); + g_test_add_func("/utils/locale-workaround", test_locale_workaround); + + return g_test_run(); +} diff --git a/man/author.inc b/man/author.inc new file mode 100644 index 0000000..db14d93 --- /dev/null +++ b/man/author.inc @@ -0,0 +1,7 @@ +* AUTHOR + +Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +# Local Variables: +# mode: org +# End: diff --git a/man/bugs.inc b/man/bugs.inc new file mode 100644 index 0000000..882e6a5 --- /dev/null +++ b/man/bugs.inc @@ -0,0 +1,7 @@ +* REPORTING BUGS + +Please report bugs at <https://github.com/djcb/mu/issues>. + +# Local Variables: +# mode: org +# End: diff --git a/man/common-options.inc b/man/common-options.inc new file mode 100644 index 0000000..ec83e3f --- /dev/null +++ b/man/common-options.inc @@ -0,0 +1,31 @@ +* COMMON OPTIONS + +** -d, --debug +makes mu generate extra debug information, useful for debugging the program +itself. By default, debug information goes to the log file, ~/.cache/mu/mu.log. +It can safely be deleted when mu is not running. When running with --debug +option, the log file can grow rather quickly. See the note on logging below. + +** -q, --quiet +causes mu not to output informational messages and progress information to +standard output, but only to the log file. Error messages will still be sent to +standard error. Note that mu index is much faster with --quiet, so it is +recommended you use this option when using mu from scripts etc. + +** --log-stderr +causes mu to not output log messages to standard error, in addition to sending +them to the log file. + +** --nocolor +do not use ANSI colors. The environment variable ~NO_COLOR~ can be used as an +alternative to ~--nocolor~. + +** -V, --version +prints mu version and copyright information. + +** -h, --help +lists the various command line options. + +# Local Variables: +# mode: org +# End: diff --git a/man/copyright.inc.in b/man/copyright.inc.in new file mode 100644 index 0000000..2e02670 --- /dev/null +++ b/man/copyright.inc.in @@ -0,0 +1,12 @@ +* COPYRIGHT + +This manpage is part of ~mu~ @VERSION@. + +Copyright © 2008-@YEAR@ Dirk-Jan C. Binnema. License GPLv3+: GNU GPL version 3 +or later <https://gnu.org/licenses/gpl.html>. This is free software: you are +free to change and redistribute it. There is NO WARRANTY, to the extent +permitted by law. + +# Local Variables: +# mode: org +# End: diff --git a/man/exit-code.inc b/man/exit-code.inc new file mode 100644 index 0000000..07c8138 --- /dev/null +++ b/man/exit-code.inc @@ -0,0 +1,14 @@ +* EXIT CODE + +This command returns 0 upon successful completion, or a non-zero exit code +otherwise. + + 0. success + 2. no matches found. Try a different query + 11. database schema mismatch. You need to re-initialize ~mu~, see *mu-init(1)* + 19. failed to acquire lock. Some other program has exclusive access to the mu database + 99. caught an exception + +# Local Variables: +# mode: org +# End: diff --git a/man/meson.build b/man/meson.build new file mode 100644 index 0000000..de4ba29 --- /dev/null +++ b/man/meson.build @@ -0,0 +1,102 @@ +## Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# generate org include files +# +man_data=configuration_data() +man_data.set('VERSION', meson.project_version()) +man_data.set('YEAR', mu_year) +incs=[ + 'author.inc', + 'bugs.inc', + 'common-options.inc', + 'copyright.inc.in', + 'exit-code.inc', + 'muhome.inc', + 'prefooter.inc', +] +foreach inc: incs + # configure the .in ones + if inc.substring(-3) == '.in' + configure_file(input: inc, + output: '@BASENAME@', + configuration: man_data) + else # and copy the rest + configure_file(input: inc, output:'@BASENAME@.inc', + copy:true) + endif +endforeach + +# man-pages is org-format. +man_orgs=[ + 'mu.1.org', + 'mu-add.1.org', + 'mu-bookmarks.5.org', + 'mu-cfind.1.org', + 'mu-easy.7.org', + 'mu-extract.1.org', + 'mu-find.1.org', + 'mu-help.1.org', + 'mu-index.1.org', + 'mu-info.1.org', + 'mu-init.1.org', + 'mu-mkdir.1.org', + 'mu-move.1.org', + 'mu-query.7.org', + 'mu-remove.1.org', + 'mu-server.1.org', + 'mu-verify.1.org', + 'mu-view.1.org' +] + +foreach src : man_orgs + # meson makes in tricky to use the results of e.g. configure_file + # in custom_commands..., so this is admittedly a little hacky. + org = join_paths(meson.current_build_dir(), src) + man = '@BASENAME@' + section = src.substring(-5, -4) + + # we fill in some man-page details: + # @SECTION_ID@: the man-page section + # @MAN_DATE@: date of the generation (not yet supported by ox-man) + conf_data = configuration_data() + conf_data.set('SECTION_ID', section) + conf_data.set('MAN_DATE', mu_month_year) + configure_file(input: src, output:'@BASENAME@.org', + configuration: conf_data) + + expr_tmpl = ''.join([ + '(progn', + ' (require \'ox-man)', + ' (setq org-export-with-sub-superscripts \'{})', + ' (org-export-to-file \'man "@0@"))']) + expr = expr_tmpl.format(org.substring(0,-4)) + sectiondir = join_paths(mandir, 'man' + section) + + custom_target(src + '-to-man', + build_by_default: true, + input: src, + output: '@BASENAME@', + install: true, + install_dir: sectiondir, + depend_files: incs, + command: [emacs, + '--no-init-file', + '--batch', + org, + '--eval', expr]) +endforeach diff --git a/man/mu-add.1.org b/man/mu-add.1.org new file mode 100644 index 0000000..f2b644c --- /dev/null +++ b/man/mu-add.1.org @@ -0,0 +1,29 @@ +#+TITLE: MU ADD +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-add - add one or more messages to the database + +* SYNOPSIS + +*mu [common-options] add [options] <file> [<files>]* + +* DESCRIPTION + +~mu add~ is the command to add specific message files to the database. Each file +must be specified with an absolute path. + +* ADD OPTIONS + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)*, *mu-index(1)*, *mu-remove(1)* diff --git a/man/mu-bookmarks.5.org b/man/mu-bookmarks.5.org new file mode 100644 index 0000000..b7d275e --- /dev/null +++ b/man/mu-bookmarks.5.org @@ -0,0 +1,36 @@ +#+TITLE: MU BOOKMARKS +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-bookmarks - file with bookmarks (shortcuts) for mu search expressions + +* DESCRIPTION + +Bookmarks are named shortcuts for search queries. They allow using a convenient +name for often-used queries. The bookmarks are also visible as shortcuts in the +mu experimental user interfaces, =mug= and =mug2=. + +The bookmarks file is read from =<muhome>/bookmarks=. On Unix this would typically +be w be =~/.config/mu/bookmarks=, but this can be influenced using the ~--muhome~ +parameter for *mu-find(1)*. + +The bookmarks file is a typical key=value *.ini*-file, which is best shown by +means of an example: + +#+begin_example +[mu] +inbox=maildir:/inbox # inbox +oldhat=maildir:/archive subject:hat # archived with subject containing 'hat' +#+end_example + +The *[mu]* group header is required. For practical uses of bookmarks, see +*mu-find(1)*. + +#+include: "author.inc" :minlevel 1 + +#+include: "copyright.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)*, *mu-find(1)* diff --git a/man/mu-cfind.1.org b/man/mu-cfind.1.org new file mode 100644 index 0000000..0c14dc6 --- /dev/null +++ b/man/mu-cfind.1.org @@ -0,0 +1,161 @@ +#+TITLE: MU CFIND +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-cfind - find contacts in the *mu* database and export them +for use in other programs. + +* SYNOPSIS + +*mu [common-options] cfind [options] [<pattern>]* + +* DESCRIPTION + +*mu cfind* is the *mu* command for finding =contacts= (name and e-mail address of +people who were either an e-mail's sender or receiver). There are different +output formats available, for importing the contacts into other programs. + +* SEARCHING CONTACTS + +When you index your messages (see *mu index*), *mu* creates a list of unique e-mail +addresses found and the accompanying name, and caches this list. In case the +same e-mail address is used with different names, the most recent non-empty name +is used. + +*mu cfind* starts a search for contacts that match a =regular expression=. For +example: + +#+begin_example +$ mu cfind '@gmail\.com' +#+end_example + +would find all contacts with a gmail-address, while + +#+begin_example +$ mu cfind Mary +#+end_example + +lists all contacts with Mary in either name or e-mail address. + +If you do not specify a search expression, *mu cfind* returns the full list of +contacts. Note, *mu cfind* uses a cache with the e-mail information, which is +populated during the indexing process. + +The regular expressions are basic case-insensitive PCRE, see *pcre(3)*. + +* CFIND OPTIONS + +** --format=plain|mutt-alias|mutt-ab|wl|org-contact|bbdb|csv +sets the output format to the given value. The following are available: + +#+ATTR_MAN: :disable-caption t +| --format= | description | +|-------------+-----------------------------------| +| plain | default, simple list | +| mutt-alias | mutt alias-format | +| mutt-ab | mutt external address book format | +| wl | wanderlust addressbook format | +| org-contact | org-mode org-contact format | +| bbdb | BBDB format | +| csv | comma-separated values [1] | +| json | JSON format | + + +[1] *CSV* is not fully standardized, but *mu cfind* follows some common practices: +any double-quote is replaced by a double-double quote (thus, "hello" become +""hello"", and fields with commas are put in double-quotes. Normally, this +should only apply to name fields. + +** --personal,-p only show addresses seen in messages where one of `my' e-mail +addresses was seen in one of the address fields; this is to exclude addresses +only seen in mailing-list messages. See the ~--my-address~ parameter to *mu init*. + +** --after=<timestamp> only show addresses last seen after +=<timestamp>=. =<timestamp>= is a UNIX *time_t* value, the number of +seconds since 1970-01-01 (in UTC). + +From the command line, you can use the *date* command to get this value. For +example, only consider addresses last seen after 2020-06-01, you could specify +#+begin_example + --after=`date +%s --date='2020-06-01'` +#+end_example + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +* JSON FORMAT + +With ~--format=json~, the matching contacts come out as a JSON array, e.g., +#+begin_example +[ + { + "email" : "syb@example.com", + "name" : "Sybil Gerard", + "display" : "Sybil Gerard <syb@example.com>", + "last-seen" : 1075982687, + "last-seen-iso" : "2004-02-05T14:04:47Z", + "personal" : false, + "frequency" : 14 + }, + { + "email" : "ed@example.com", + "name" : "Mallory, Edward", + "display" : "\"Mallory, Edward\" <ed@example.com>", + "last-seen" : 1425991805, + "last-seen-iso" : "2015-03-10T14:50:05Z", + "personal" : true, + "frequency" : 2 + } +] +#+end_example + +Each contact has the following fields: + +#+ATTR_MAN: :disable-caption t +| property | description | +|---------------+--------------------------------------------------------------------------| +| ~email~ | the email-address | +| ~name~ | the name (or ~none~) | +| ~display~ | the combination name and e-mail address for display purposes | +| ~last-seen~ | date of most recent message with this contact (Unix time) | +| ~last-seen-iso~ | ~last-seen~ represented as an ISO-8601 timestamp | +| ~personal~ | whether the email was seen in a message together with a personal address | +| ~frequency~ | approximation of the number of times this contact was seen in messages | + +The JSON format is useful for further processing, e.g. using the *jq(1)* tool: + +List display names, sorted by their last-seen date: +#+begin_example +$ mu cfind --format=json --personal | jq -r '.[] | ."last-seen-iso" + " " + .display' | sort +#+end_example + +* INTEGRATION WITH MUTT + +You can use *mu cfind* as an external address book server for *mutt*. +For this to work, add the following to your =muttrc=: + +#+begin_example +set query_command = "mu cfind --format=mutt-ab '%s'" +#+end_example + +Now, in mutt, you can search for e-mail addresses using the *query*-command, +which is (by default) accessible by pressing *Q*. + +* ENCODING + +*mu cfind* output is encoded according to the current locale except for +=--format=bbdb=. This is hard-coded to UTF-8, and as such specified in the +output-file, so emacs/bbdb can handle things correctly, without guessing. + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "bugs.inc" :minlevel 1 + +#+include: "author.inc" :minlevel 1 + +#+include: "copyright.inc" :minlevel 1 + +* SEE ALSO +*mu(1)*, *mu-index(1)*, *mu-find(1)*, *pcre(3)*, *jq(1)* diff --git a/man/mu-easy.7.org b/man/mu-easy.7.org new file mode 100644 index 0000000..662a314 --- /dev/null +++ b/man/mu-easy.7.org @@ -0,0 +1,303 @@ +#+TITLE: MU EASY +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-easy - a quick introduction to mu + +* DESCRIPTION + +*mu* is a set of tools for dealing with e-mail messages in Maildirs. There are +many options, which are all described in the man pages for the various +sub-commands. This man pages jumps over all of the details and gives examples of +some common use cases. If the use cases described here do not precisely do what +you want, please check the more extensive information in the man page about the +sub-command you are using -- for example, the *mu-index(1)* or *mu-find(1)* man +pages. + +*NOTE*: the *index* command (and therefore, the ones that depend on that, such as +*find*), require that you store your mail in the Maildir-format. If you don't do +so, you can still use the other commands, but you won't be able to index/search +your mail. + +By default, *mu* uses colorized output when it thinks your terminal is capable of +doing so. If you don't like color, you can use the *--nocolor* command-line +option, or set either the *MU_NOCOLOR* or the *NO_COLOR* environment variable to +non-empty. + +* SETTING THINGS UP + +The first time you run the mu commands, you need to initialize it. This is done +with the *init* command. + +#+begin_example +$ mu init +#+end_example + +This uses the defaults (see *mu-init(1)* for details on how to change that). + + +* INDEXING YOUR E-MAIL + +Before you can search e-mails, you'll first need to index them: + +#+begin_example +$ mu index +#+end_example + +The process can take a few minutes, depending on the amount of mail you have, +the speed of your computer, hard drive etc. Usually, indexing should be able to +reach a speed of a few hundred messages per second. + +*mu index* guesses the top-level Maildir to do its job; if it guesses wrong, you +can use the =--maildir= option to specify the top-level directory that should be +processed. See the *mu-index(1)* man page for more details. + +Normally, *mu index* visits all the directories under the top-level Maildir; +however, you can exclude certain directories (say, the `trash' or `spam' +folders) by creating a file called =.noindex= in the directory. When *mu* sees such +a file, it will exclude this directory and its sub-directories from indexing. +Also see *.noupdate* in the *mu-index(1)* manpage. + +* SEARCHING YOUR E-MAIL + +After you have indexed your mail, you can start searching it. By default, the +search results are printed on standard output. Alternatively, the output can +take the form of Maildir with symbolic links to the found messages. This enables +integration with e-mail clients; see the *mu-find(1)* man page for details, the +syntax of the search parameters and so on. Here, we just give some examples for +common cases. + +You can use the *mu fields* command to get information about all possible fields +and flags. + +First, let's search for all messages sent to Julius (Caesar) regarding fruit: + +#+begin_example +$ mu find t:julius fruit +#+end_example + +This should return something like: + +#+begin_example +2008-07-31T21:57:25 EEST John Milton <jm@example.com> Fere libenter homines id quod volunt credunt +#+end_example + +This means there is a message to `julius' with `fruit' somewhere in the message. +In this case, it's a message from John Milton. Note that the date format depends +on your the language/locale you are using. + +How do we know that the message was sent to Julius Caesar? Well, it's not +visible from the results above, because the default fields that are shown are +date/sender/subject. However, we can change this using the =--fields= parameter +(try *mu fields* to see all the details): + +#+begin_example +$ mu find --fields="t s" t:julius fruit +#+end_example + +In other words, display the `To:'-field (t) and the subject (s). This should +return something like: +#+begin_example +Julius Caesar <jc@example.com> Fere libenter homines id quod volunt credunt +#+end_example + +This is the same message found before, only with some different fields +displayed. + +By default, *mu* uses the logical ~AND~ for the search parameters -- that is, it +displays messages that match all the parameters. However, we can use logical ~OR~ +as well: + +#+begin_example +$ mu find t:julius OR f:socrates +#+end_example + +In other words, display messages that are either sent to Julius Caesar *or* are +from Socrates. This could return something like: + +#+begin_example +2008-07-31T21:57:25 EEST Socrates <soc@example.com> cool stuff +2008-07-31T21:57:25 EEST John Milton <jm@example.com> Fere libenter homines id quod volunt credunt +#+end_example + +What if we want to see some of the body of the message? You can get a `summary' +of the first lines of the message using the =--summary-len= option, which will +`summarize' the first =n= lines of the message: + +#+begin_example +$ mu find --summary-len=3 napoleon m:/archive +#+end_example + +#+begin_example +1970-01-01T02:00:00 EET Napoleon Bonaparte <nb@example.com> rock on dude +Summary: Le 24 février 1815, la vigie de Notre-Dame de la Garde signala le +trois-mâts le Pharaon, venant de Smyrne, Trieste et Naples. Comme +d'habitude, un pilote côtier partit aussitôt du port, rasa le château +#+end_example + +The summary consists of the first /n/ lines of the message with all superfluous +whitespace removed. + +Also note the *m:/archive* parameter in the query. This means that we only match +messages in a maildir called ~'/archive'~. + +* MORE QUERIES + +Let's list a few more queries that may be interesting; please note that +searches for message flags, priority and date ranges are only available in mu +version 0.9 or later. + +Get all important messages which are signed: +#+begin_example + *$ mu find flag:signed prio:high * +#+end_example + +Get all messages from Jim without an attachment: +#+begin_example + *$ mu find from:jim AND NOT flag:attach* +#+end_example + +Get all messages where Jack is in one of the contact fields: +#+begin_example + *$ mu find contact:jack* +#+end_example +This uses the special contact: pseudo-field which matches (*from*, +*to*, *cc* and *bcc*). + +Get all messages in the Sent Items folder about yoghurt: +#+begin_example + *$mu find maildir:'/Sent Items' yoghurt* +#+end_example +Note how we need to quote search terms that include spaces. + + +Get all unread messages where the subject mentions Ångström: +#+begin_example + *$ mu find subject:Ångström flag:unread* +#+end_example +which is equivalent to: +#+begin_example + *$ mu find subject:angstrom flag:unread* +#+end_example +because does mu is case-insensitive and accent-insensitive. + +Get all unread messages between March 2002 and August 2003 about some bird (or +a Swedish rock band): +#+begin_example + *$ mu find date:20020301..20030831 nightingale flag:unread* +#+end_example + +Get all messages received today: +#+begin_example + *$ mu find date:today..now* +#+end_example + +Get all messages we got in the last two weeks about emacs: +#+begin_example + *$ mu find date:2w..now emacs* +#+end_example + +Another powerful feature (since 0.9.6) are wildcard searches, where you can +search for the last =n= characters in a word. For example, you can search +for: +#+begin_example + *$ mu find 'subject:soc*'* +#+end_example +and get mails about soccer, Socrates, society, and so on. Note, it's important +to quote the search query, otherwise the shell will interpret +the `*'. + +You can also search for messages with a certain attachment using their +filename, for example: + +#+begin_example + *$ mu find 'file:pic*'* +#+end_example +will get you all messages with an attachment starting with `pic'. + +If you want to find attachments with a certain MIME-type, you can use the +following: + +Get all messages with PDF attachments: +#+begin_example + *$ mu find mime:application/pdf* +#+end_example + +or even: + +Get all messages with image attachments: +#+begin_example + *$ mu find 'mime:image/*'* +#+end_example + + +Note that (1) the `*' wildcard can only be used as the rightmost thing in a +search query, and (2) that you need to quote the search term, because +otherwise your shell will interpret the `*' (expanding it to all files in the +current directory -- probably not what you want). + +* DISPLAYING MESSAGES + +We might also want to display the complete messages instead of the header +information. This can be done using *mu view* command. Note that this +command does not use the database; you simply provide it the path to a +message. + +Therefore, if you want to display some message from a search query, you'll +need its path. To get the path (think *l*ocation) for our first example we +can use: + +#+begin_example +$ mu find --fields="l" t:julius fruit +#+end_example + +And we'll get something like: +#+begin_example +/home/someuser/Maildir/archive/cur/1266188485_0.6850.cthulhu:2, +#+end_example + +We can now display this message: + +#+begin_example +$ mu view /home/someuser/Maildir/archive/cur/1266188485_0.6850.cthulhu:2, +From: John Milton <jm@example.com> +To: Julius Caesar <jc@example.com> +Subject: Fere libenter homines id quod volunt credunt +Date: 2008-07-31T21:57:25 EEST + +OF Mans First Disobedience, and the Fruit +Of that Forbidden Tree, whose mortal taste +Brought Death into the World, and all our woe, +[...] +#+end_example + +* FINDING CONTACTS + +While *mu find* searches for messages, there is also *mu cfind* to find =contacts=, +that is, names + addresses. Without any search expression, *mu cfind* lists all of +your contacts. + +#+begin_example +$ mu cfind julius +#+end_example + +will find all contacts with `julius' in either name or e-mail address. Note that +*mu cfind* accepts a =regular expression= (as per *pcre(3)*) + +*mu cfind* also supports a =--format==-parameter, which sets the output to some +specific format, so the results can be imported into another program. For +example, to export your contact information to a *mutt* address book file, you can +use something like: + +#+begin_example +$ mu cfind --format=mutt-alias > ~/mutt-aliases +#+end_example + +Then, you can use them in *mutt* if you add something like *source ~/mutt-aliases* +to your =muttrc=. + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO +*mu(1)*, *mu-init(1)*, *mu-index(1)*, *mu-find(1)*, *mu-mfind(1)*, *mu-mkdir(1)*, *mu-view(1)*, *mu-extract(1)* diff --git a/man/mu-extract.1.org b/man/mu-extract.1.org new file mode 100644 index 0000000..596a663 --- /dev/null +++ b/man/mu-extract.1.org @@ -0,0 +1,108 @@ +#+TITLE: MU EXTRACT +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-extract - display and save message parts +(attachments), and open them with other tools. + +* SYNOPSIS + +*mu [common-options] extract [options] [<file>]* + +*mu [common-options] extract [options] <file> <pattern>* + +* DESCRIPTION + +*mu extract* is the *mu* sub-command for extracting MIME-parts (e.g., attachments) +from mail messages. The sub-command works on message files, and does not require +the message to be indexed in the database. + +For attachments, the file name used when saving it is the name of the attachment +in the message. If there is no such name, or when saving non-attachment +MIME-parts, a name is derived from the message-id of the message. + +If you specify a regular express pattern as the second argument, all attachments +with filenames matching that pattern will be extracted. The regular expressions +are basic PCRE, and are case-sensitive by default; see *pcre(3)* for more details. + +Without any options, *mu extract* simply outputs the list of leaf MIME-parts in +the message. Only `leaf' MIME-parts (including RFC822 attachments) are +considered, *multipart/** etc. are ignored. + +Without a filename parameter, ~mu extract~ reads a message from standard-input. In +that case, you cannot use the second, ~<pattern>~ parameter as this would be +ambiguous; instead, use the ~--matches~ option. + +* EXTRACT OPTIONS + +** -a, --save-attachments +save all MIME-parts that look like attachments. + +** --save-all +save all non-multipart MIME-parts. + +** --parts=<parts> +only consider the following numbered parts (comma-separated list). The numbers +for the parts can be seen from running *mu extract* without any options but only +the message file. + +** --target-dir=<dir> +save the parts in the target directory rather than the current working +directory. + +** --overwrite +overwrite existing files with the same name; by default overwriting is not +allowed. + +** -u,--uncooked +by default, ~mu~ transforms the attachment filenames a bit (such as by replacing +spaces by dashes); with this option, leave that to the minimum for creating +a legal filename in the target directory. + +** --matches=<pattern> +Attachments with filenames matching the pattern will be extracted. The regular +expressions are basic PCRE, and are case-sensitive by default; see *pcre(3)* for +more details. + +** --play +Try to `play' (open) the attachment with the default application for the +particular file type. On MacOS, this uses the *open* program, on other platforms +it uses *xdg-open*. You can choose a different program by setting the +*MU_PLAY_PROGRAM* environment variable. + +#+include: "common-options.inc" :minlevel 1 + +* EXAMPLES + +To display information about all the MIME-parts in a message file: +#+begin_example +$ mu extract msgfile +#+end_example + +To extract MIME-part 3 and 4 from this message, overwriting existing files with +the same name: +#+begin_example +$ mu extract --parts=3,4 --overwrite msgfile +#+end_example + +To extract all files ending in `.jpg' (case-insensitive): +#+begin_example +$ mu extract msgfile '.*\.jpg' +#+end_example + +To extract an mp3-file, and play it in the default mp3-playing application: +#+begin_example +$ mu extract --play msgfile 'whoopsididitagain.mp3' +#+end_example + +when reading from standard-input, you need ~--matches~, so: +#+begin_example +$ cat msgfile | mu extract --play --matches 'whoopsididitagain.mp3' +#+end_example + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)* diff --git a/man/mu-find.1.org b/man/mu-find.1.org new file mode 100644 index 0000000..a8fc9fe --- /dev/null +++ b/man/mu-find.1.org @@ -0,0 +1,314 @@ +#+TITLE: MU FIND +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-find - find e-mail messages in the *mu* database. + +* SYNOPSIS + +*mu [common-options] find [options] <search expression>* + +* DESCRIPTION + +*mu find* is the *mu* command for searching e-mail message that were stored earlier +using *mu index(1)*. + +* SEARCHING MAIL + +*mu find* starts a search for messages in the database that match some search +pattern. The search patterns are described in detail in *mu-query(7)*. + +For example: + +#+begin_example +$ mu find subject:snow and date:2009.. +#+end_example + +would find all messages in 2009 with `snow' in the subject field, e.g: + +#+begin_example +2009-03-05 17:57:33 EET Lucia <lucia@example.com> running in the snow +2009-03-05 18:38:24 EET Marius <marius@foobar.com> Re: running in the snow +#+end_example + +Note, this the default, plain-text output, which is the default, so you don't +have to use *--format=plain*. For other types of output (such as symlinks, XML or +s-expressions), see the discussion in the *OPTIONS*-section below about *--format*. + +The search pattern is taken as a command-line parameter. If the search +parameter consists of multiple parts (as in the example) they are +treated as if there were a logical *and* between them. + +For details on the possible queries, see *mu-query(7)*. + +* FIND OPTIONS + +Note, some of the important options are described in the *mu*(1) man-page +and not here, as they apply to multiple mu-commands. + +The *find*-command has various options that influence the way *mu* displays the +results. If you don't specify anything, the defaults are ~fields="d f s"~, +~--sortfield=date~ and ~--reverse~. + +** -f, --fields=<fields> +specifies a string that determines which fields are shown in the output. This +string consists of a number of characters (such as 's' for subject or 'f' for +from), which will replace with the actual field in the output. Fields that are +not known will be output as-is, allowing for some simple formatting. + +For example: + +#+begin_example +$ mu find subject:snow --fields "d f s" +#+end_example + +lists the date, subject and sender of all messages with `snow' in the their +subject. + +The table of replacement characters is superset of the list mentions for search +parameters, such as: +#+begin_example + t *t*o: recipient + d Sent *d*ate of the message + f Message sender (*f*rom:) + g Message flags (fla*g*s) + l Full path to the message (*l*ocation) + s Message *s*ubject + i Message-*i*d + m *m*aildir +#+end_example + +For the complete list, try the command: ~mu info fields~. + +The message flags are described in *mu-query(7)*. As an example, a message which +is `seen', has an attachment and is signed would have `asz' as its corresponding +output string, while an encrypted new message would have `nx'. + +** -s, --sortfield=<field> and -z,--reverse +specify the field to sort the search results by and the direction (i.e., +`reverse' means that the sort should be reverted - Z-A). Examples include: + +#+begin_example + cc,c Cc (carbon-copy) recipient(s) + date,d Message sent date + from,f Message sender + maildir,m Maildir + msgid,i Message id + prio,p Nessage priority + subject,s Message subject + to,t To:-recipient(s) +#+end_example + +For the complete list, try the command: ~mu info fields~. + +Thus, for example, to sort messages by date, you could specify: + +#+begin_example +$ mu find fahrrad --fields "d f s" --sortfield=date --reverse +#+end_example + +Note, if you specify a sortfield, by default, messages are sorted in reverse +(descending) order (e.g., from lowest to highest). This is usually a good +choice, but for dates it may be more useful to sort in the opposite direction. + +** -n, --maxnum=<number> +If > 0, display maximally that number of entries. If not specified, all matching +entries are displayed. + +** --summary-len=<number> +If > 0, use that number of lines of the message to provide a summary. + +** --format=<plain|links|xml|sexp> + +output results in the specified format: + +- The default is *plain*, i.e normal output with one line per message. +- *links* outputs the results as a maildir with symbolic links to the found + messages. This enables easy integration with mail-clients (see below for more + information). +- *xml* formats the search results as XML. +- *sexp* formats the search results as an s-expression as used in Lisp programming + environments + +** --linksdir=<dir> and -c, --clearlinks +when using ~-format=links~, output the results as a maildir with symbolic links to +the found messages. This enables easy integration with mail-clients (see below +for more information). *mu* will create the maildir if it does not exist yet. + +If you specify ~--clearlinks~, existing symlinks will be cleared from the target +directories; this allows for re-use of the same maildir. However, this option +will delete any symlink it finds, so be careful. + +#+begin_example +$ mu find grolsch --format=links --linksdir=~/Maildir/search --clearlinks +#+end_example + +stores links to found messages in =~/Maildir/search=. If the directory does not +exist yet, it will be created. Note: when *mu* creates a Maildir for these links, +it automatically inserts a =.noindex= file, to exclude the directory from *mu +index*. + +** --after=<timestamp> +only show messages whose message files were last modified (*mtime*) after +=<timestamp>=. =<timestamp>= is a UNIX *time_t* value, the number of seconds since +1970-01-01 (in UTC). + +From the command line, you can use the *date* command to get this value. For +example, only consider messages modified (or created) in the last 5 minutes, you +could specify +#+begin_example + --after=`date +%s --date='5 min ago'` +#+end_example +This is assuming the GNU *date* command. + +** --exec=<command> +the ~--exec~ coption causes the =command= to be executed on each matched message; +for example, to see the raw text of all messages matching `milkshake', you could +use: +#+begin_example +$ mu find milkshake --exec='less' +#+end_example +which is roughly equivalent to: +#+begin_example +$ mu find milkshake --fields="l" | xargs less +#+end_example + +** -b, --bookmark=<bookmark> +use a bookmarked search query. Using this option, a query from your bookmark +file will be prepended to other search queries. See *mu-bookmarks(5)* for the +details of the bookmarks file. + + +** -u, --skip-dups +whenever there are multiple messages with the same message-id field, only show +the first one. This is useful if you have copies of the same message, which is a +common occurrence when using e.g. Gmail together with *offlineimap*. + +** -r, --include-related +include messages being referred to by the matched messages -- i.e.. include +messages that are part of the same message thread as some matched messages. This +is useful if you want Gmail-style `conversations'. + +** -t, --threads +show messages in a `threaded' format -- that is, with indentation and arrows +showing the conversation threads in the list of matching messages. When using +this, sorting is chronological (by date), based on the newest message in a +thread. + +Messages in the threaded list are indented based on the depth in the discussion, +and are prefix with a kind of arrow with thread-related information about the +message, as in the following table: +#+begin_example +| | normal | orphan | duplicate | +|-------------+--------+--------+-----------| +| first child | `-> | `*> | `=> | +| other | |-> | |*> | |=> | +#+end_example + +Here, an `orphan' is a message without a parent message (in the list of +matches), and a duplicate is a message whose message-id was already seen before; +not this may not really be the same message, if the message-id was copied. + +The algorithm used for determining the threads is based on Jamie Zawinksi's +description: http://www.jwz.org/doc/threading.html + +** -a,--analyze +instead of executing the query, analyze it by show the parse-tree s-expression +and a stringified version of the Xapian query. This can help users to determine +how ~mu~ interprets some query. + +The output of this command are differ between versions, but should be helpful +nevertheless. + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +* INTEGRATION + +It is possible to integrate *mu find* with some mail clients + +** *mutt* + +For *mutt* you can use the following in your =muttrc=; pressing the F8 key will +start a search, and F9 will take you to the results. + +#+begin_example +# mutt macros for mu +macro index <F8> "<shell-escape>mu find --clearlinks --format=links --linksdir=~/Maildir/search " \\ + "mu find" +macro index <F9> "<change-folder-readonly>~/Maildir/search" \\ + "mu find results" +#+end_example + + +** *Wanderlust* + +*Sam B* suggested the following on the *mu*-mailing list. First add the following to +your Wanderlust configuration file: + +#+begin_example +(require 'elmo-search) +(elmo-search-register-engine + 'mu 'local-file + :prog "/usr/local/bin/mu" ;; or wherever you've installed it + :args '("find" pattern "--fields" "l") :charset 'utf-8) + +(setq elmo-search-default-engine 'mu) +;; for when you type "g" in folder or summary. +(setq wl-default-spec "[") +#+end_example + +Now, you can search using the *g* key binding; you can also create permanent +virtual folders when the messages matching some expression by adding something +like the following to your =folders= file. + +#+begin_example +VFolders { + [date:today..now]!mu "Today" + [size:1m..100m]!mu "Big" + [flag:unread]!mu "Unread" +} +#+end_example + +After restarting Wanderlust, the virtual folders should appear. + +* ENCODING + +*mu find* output is encoded according to the locale for =--format=plain= (the +default format), and UTF-8 for all other formats (=sexp=, =xml=). + + +* PERFORMANCE + +Some notes on performance, comparing the timings between some recent releases; +taking the total number for 10 test runs. + +1. time (repeat 10 mu find "" -n 50000 > /dev/null) +2. time (repeat 10 mu find "" -n 50000 --include-related --threads > /dev/null) + + +#+ATTR_MAN: :disable-caption t +| release | time 1 (sec) | time 2 (sec) | +|---------------+--------------+--------------| +| 1.4 | 8.9s | 59.3s | +| 1.6 | 8.3s | 27.5s | +| 1.8 | 8.7s | 29.3s | +| 1.10 | 9.8s | 30.6s | +| 1.11 (master) | 10.1s | 29.5s | + + + + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "bugs.inc" :minlevel 1 + +#+include: "author.inc" :minlevel 1 + +#+include: "copyright.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)*, *mu-index(1)*, *mu-query(7)*, *mu-info(1)* diff --git a/man/mu-help.1.org b/man/mu-help.1.org new file mode 100644 index 0000000..2a67dc2 --- /dev/null +++ b/man/mu-help.1.org @@ -0,0 +1,20 @@ +#+TITLE: MU HELP +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-help - show help information about mu commands. + +* SYNOPSIS + +*mu [common-options] help [<command>]* + +* DESCRIPTION + +*mu help* provides help information about mu commands. + +#+include: "common-options.inc" :minlevel 1 + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 diff --git a/man/mu-index.1.org b/man/mu-index.1.org new file mode 100644 index 0000000..80452e0 --- /dev/null +++ b/man/mu-index.1.org @@ -0,0 +1,218 @@ +#+TITLE: MU INDEX +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-index - index e-mail messages stored in Maildirs + +* SYNOPSIS + +*mu [common-options] index* + +* DESCRIPTION + +*mu index* is the *mu* command for scanning the contents of Maildir directories and +storing the results in a Xapian database. The data can then be queried using +*mu-find(1)*. + +Before the first time you run *mu index*, you must run *mu init* to initialize the +database. + +*index* understands Maildirs as defined by Daniel Bernstein for *qmail(7)*. In +addition, it understands recursive Maildirs (Maildirs within Maildirs), +Maildir++. It also supports VFAT-based Maildirs which use =!= or =;= as the +separators instead of =:=. + +E-mail messages which are not stored in something resembling a maildir +leaf-directory (=cur= and =new=) are ignored, as are the cache directories for +=notmuch= and =gnus=, and any dot-directory. + +Symlinks are followed, and the directories can be spread over multiple +filesystems; however note that moving files around is much faster when multiple +filesystems are not involved. Be careful to avoid self-referential symlinks! + +If there is a file called =.noindex= in a directory, the contents of that +directory and all of its subdirectories will be ignored. This can be useful to +exclude certain directories from the indexing process, for example directories +with spam-messages. + +If there is a file called =.noupdate= in a directory, the contents of that +directory and all of its subdirectories will be ignored. This can be useful to +speed up things you have some maildirs that never change. + +=.noupdate= does not affect already-indexed message: you can still search for +them. =.noupdate= is ignored when you start indexing with an empty database (such +as directly after =mu init=). + +There also the option *--lazy-check* which can greatly speed up indexing; see +below for details. + +The first run of *mu index* may take a few minutes if you have a lot of mail (tens +of thousands of messages). Fortunately, such a full scan needs to be done only +once; after that it suffices to index the changes, which goes much faster. See +the `PERFORMANCE (i,ii,iii)' below for more information. + +The optional `phase two' of the indexing-process is the removal of messages from +the database for which there is no longer a corresponding file in the Maildir. +If you do not want this, you can use ~-n~, ~--nocleanup~. + +When *mu index* catches one of the signals *SIGINT*, *SIGHUP* or *SIGTERM* (e.g., when +you press Ctrl-C during the indexing process), it attempts to shutdown +gracefully; it tries to save and commit data, and close the database etc. If it +receives another signal (e.g., when pressing Ctrl-C once more), *mu index* will +terminate immediately. + +* INDEX OPTIONS + +** --lazy-check + +in lazy-check mode, *mu* does not consider messages for which the time-stamp +(ctime) of the directory they reside in has not changed since the previous +indexing run. This is much faster than the non-lazy check, but won't update +messages that have change (rather than having been added or removed), since +merely editing a message does not update the directory time-stamp. Of course, +you can run *mu-index* occasionally without ~--lazy-check~, to pick up such +messages. + +** --nocleanup + +disable the database cleanup that *mu* does by default after indexing. + +** --reindex + +perform a complete reindexing of all the messages in the maildir. + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +* ENCRYPTION + +*mu index* does _not_ decrypt messages, and only the metadata (such as headers) of +encrypted messages makes it to the database. *mu view* and *mu4e* can decrypt +messages, but those work with the message directly and the information is not +added to the database. + +* PERFORMANCE + +** indexing in ancient times (2009?) + +As a non-scientific benchmark, a simple test on the author's machine (a Thinkpad +X61s laptop using Linux 2.6.35 and an ext3 file system) with no existing +database, and a maildir with 27273 messages: + +#+begin_example +$ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +$ time mu index --quiet +66,65s user 6,05s system 27% cpu 4:24,20 total +#+end_example +(about 103 messages per second) + +A second run, which is the more typical use case when there is a database +already, goes much faster: + +#+begin_example +$ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +$ time mu index --quiet +0,48s user 0,76s system 10% cpu 11,796 total +#+end_example +(more than 56818 messages per second) + +Note that each test flushes the caches first; a more common use case might be to +run *mu index* when new mail has arrived; the cache may stay quite `warm' in that +case: + +#+begin_example + $ time mu index --quiet + 0,33s user 0,40s system 80% cpu 0,905 total +#+end_example +which is more than 30000 messages per second. + +** indexing in 2012 + +As per June 2012, we did the same non-scientific benchmark, this time with an +Intel i5-2500 CPU @ 3.30GHz, an ext4 file system and a maildir with 22589 +messages. We start without an existing database. + +#+begin_example + $ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' + $ time mu index --quiet + 27,79s user 2,17s system 48% cpu 1:01,47 total +#+end_example +(about 813 messages per second) + +A second run, which is the more typical use case when there is a database +already, goes much faster: + +#+begin_example +$ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +$ time mu index --quiet +0,13s user 0,30s system 19% cpu 2,162 total +#+end_example +(more than 173000 messages per second) + +** indexing in 2016 + +As per July 2016, we did the same non-scientific benchmark, again with the Intel +i5-2500 CPU @ 3.30GHz, an ext4 file system. This time, the maildir contains +72525 messages. + +#+begin_example +$ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +$ time mu index --quiet +40,34s user 2,56s system 64% cpu 1:06,17 total +#+end_example +(about 1099 messages per second). + +** indexing in 2022 + +A few years later and it is June 2022. There's a lot more happening during +indexing, but indexing became multi-threaded and machines are faster; e.g. this +is with an AMD Ryzen Threadripper 1950X (16 cores) @ 3.399GHz. + +The instructions are a little different since we have a proper repeatable +benchmark now. After building, + +#+begin_example + $ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +% THREAD_NUM=4 build/lib/tests/bench-indexer -m perf +# random seed: R02Sf5c50e4851ec51adaf301e0e054bd52b +1..1 +# Start of bench tests +# Start of indexer tests +indexed 5000 messages in 20 maildirs in 3763ms; 752 μs/message; 1328 messages/s (4 thread(s)) +ok 1 /bench/indexer/4-cores +# End of indexer tests +# End of bench tests +#+end_example + +Things are again a little faster, even though the index does a lot more now +(text-normalizatian, and pre-generating message-sexps). A faster machine helps, +too! + +** recent releases + +Indexing the the same 93000-message mail corpus with the last few releases: + +#+ATTR_MAN: :disable-caption t +| release | time (sec) | notes | +|---------------+------------+------------------------------------------| +| 1.4 | 160s | | +| 1.6 | 178s | | +| 1.8 | 97s | | +| 1.10 | 120s | adds html indexing, sexp-caching | +| 1.11 (master) | 96s | adds language-guessing, batch-size=50000 | +| | | | + +Quite some variation! + +Over time new features / refactoring can change the timings quite a bit. At +least for now, the latest code is both the fastest and the most featureful! + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" + +* SEE ALSO + +*maildir(5)*, *mu(1)*, *mu-init(1)*, *mu-find(1)*, *mu-cfind(1)* diff --git a/man/mu-info.1.org b/man/mu-info.1.org new file mode 100644 index 0000000..0d182d7 --- /dev/null +++ b/man/mu-info.1.org @@ -0,0 +1,32 @@ +#+TITLE: MU INFO +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-info - show information + +* SYNOPSIS + +*mu [common options] info [<topic>]* + +* DESCRIPTION + +~mu info~ is the ~mu~ command for getting information about various topics: + +- *mu*: general mu build information (default) +- *store*: information about the message store +- *fields*: table with all the query fields and flags +- *maildirs*: list all maildirs under the store's root-maildir + +Note that while running (e.g. ~mu4e~), some of the ~store~ information can be +delayed due to database caching. + +#+include: "common-options.inc" :minlevel 1 + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)* diff --git a/man/mu-init.1.org b/man/mu-init.1.org new file mode 100644 index 0000000..6c3d4c9 --- /dev/null +++ b/man/mu-init.1.org @@ -0,0 +1,100 @@ +#+TITLE: MU INIT +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-init - initialize the mu message database + +* SYNOPSIS + +*mu [common-options] init [options]* + +* DESCRIPTION + +*mu init* is the subcommand for setting up the mu message database. After *mu init* +has completed, you can run *mu index* + +* INIT OPTIONS + +** -m, --maildir=<maildir> + +use =<maildir>= as the root-maildir. + +By default, *mu* uses the *MAILDIR* environment; if it is not set, it uses =~/Maildir= +if it is an existing directory. If neither of those can be used, the ~--maildir~ +option is required; it must be an absolute path (but ~~/~ expansion is +performed). + +** --my-address=<email-address-or-regex> + +specifies that some e-mail address is `my-address' (the option can be used +multiple times). Any message in which at least one of the contact fields +contains such an address is considered a `personal' messages; this can then be +used for filtering in *mu-find(1)*, *mu-cfind(1)* and *mu4e*, e.g. to filter-out +mailing list messages. + +=<email-address-or-regex>= can be either a plain e-mail address (such as +*foo@example.com*), or a basic PCRE regular-expression (see *pcre(3)* for details), +wrapped in */* (such as =/foo-.*@example\\.com/=). Depending on your shell, the +argument may need to be quoted. + +** --ignored-address=<email-address-or-regex> + +specifies that some e-mail address is to be ignored from the contacts-cache (the +option can be used multiple times). Such addresses then cannot be found with +*mu-cfind(1)* or in the Mu4e contacts cache. + +=<my-email-address>= can be either a plain e-mail address or a regexp, just like +for the =--my-address= option. + +** --max-message-size=<size> + +specifies the maximum size for an e-mail message. Usually, the default of +100000000 bytes should be fine. + +** --batch-size=<size> + +the number of changes after which they are committed to the database; decreasing +the value reduces the memory requirements, at the cost of make indexing +substantially slower. Usually, the default of 250000 should be fine. + +Batch-size 0 is interpreted as `use the default'. + +** --support-ngrams + +whether to enable support for using ngrams in indexing and query parsing; this +can be useful for languages without explicit word breaks, such as +Chinese/Japanese/Korean. See *NGRAM SUPPORT* below for details. + +** --reinit + +reinitialize the database from an earlier version; that is, create a new empty +database with the existing settings. This cannot be combined with the other ~init~ +options. + +#+include: "muhome.inc" :minlevel 2 + +* NGRAM SUPPORT + +*mu*'s underlying Xapian database supports `ngrams', which improve searching for +languages/scripts that do not have explicit word breaks, such as Chinese, +Japanese and Korean. It is fairly intrusive, and influences both indexing and +query-parsing; it is not enabled by default, and is recommended only if you need +to search for messages written in such languages. + +When enabled, *mu* automatically uses ngrams automatically. Xapian environment +variables such as ~XAPIAN_CJK_NGRAM~ are ignored. + +#+include: "exit-code.inc" :minlevel 1 + + +* EXAMPLE +#+begin_example +$ mu init --maildir=~/Maildir --my-address=alice@example.com --my-address=bob@example.com --ignored-address='/.*reply.*/' +#+end_example + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu-index(1)*, *mu-find(1)*, *mu-cfind(1)*, *pcre(3)* diff --git a/man/mu-mkdir.1.org b/man/mu-mkdir.1.org new file mode 100644 index 0000000..f10a714 --- /dev/null +++ b/man/mu-mkdir.1.org @@ -0,0 +1,42 @@ +#+TITLE: MU MKDIR +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-mkdir - create a new Maildir + +* SYNOPSIS + +*mu [common-options] mkdir [options] <dir> [<dirs>]* + +* DESCRIPTION + +*mu mkdir* is the command for creating Maildirs as per *maildir(5)*. A maildir is a +a directory with subdirectories ~new~, ~cur~ and ~tmp~. + +The command does not use the mu database. + +If creation fails for any reason, *no* attempt is made to remove any parts that +were created. This is for safety reasons. + +* MKDIR OPTIONS + +** --mode=<mode> +set the file access mode for the new maildir(s) as in *chmod(1)*. The default +is 0755. + +#+include: "common-options.inc" :minlevel 1 + +* EXAMPLE + +#+begin_example +$ mu mkdir tom dick harry +#+end_example + +creates three maildirs, =tom=, =dick= and =harry=. + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*maildir(5)*, *chmod(1)* diff --git a/man/mu-move.1.org b/man/mu-move.1.org new file mode 100644 index 0000000..d43a3fa --- /dev/null +++ b/man/mu-move.1.org @@ -0,0 +1,117 @@ +#+TITLE: MU MOVE +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-move - move a message file or change its flags + +* SYNOPSIS + +*mu [common-options] move [options] <src> [--flags=<flags>] [<target>]* + +* DESCRIPTION + +*mu move* is the command for moving messages in a Maildir or changing their flags. + +For any change, both the message file in the file system as well as its +representation in the database are updated accordingly. + +The source message file and target-maildir must reside under the root-maildir +for mu's database (see *mu info store*). + +* MOVE OPTIONS + +** --flags=<flags> + +specify the new message flags. See *FLAGS* for details. + +** --change-name + +change the basename of the message file when moving; this can be useful when +using some external tools such as *mbsync(1)* which otherwise get confused + +** --update-dups + +update the flags of duplicate messages too, where "duplicate messages" are +defined as all message that share the same message-id. Note that the +Draft/Flagged/Trashed flags are deliberately _not_ changed if you change those on +the source message. + +** --dry-run,-n + +print the target filename(s), but don't change anything. + +Note that with the ~--change-name~, the target name is not constant, so you cannot +use a dry-run to predict the exact name when doing a `real' run. + +#+include: "common-options.inc" :minlevel 1 + +* FLAGS + +(Note: if you are not familiar with Maildirs, please refer to the *maildir(5)* +man-page, or see http://cr.yp.to/proto/maildir.html) + +The message flags specify the Maildir-metadata for a message and are represented +by uppercase letters at the end of the message file name for all `non-new' +messages, i.e. messages that live in the ~cur/~ sub-directory of a Maildir. + +#+ATTR_MAN: :disable-caption t +| Flag | Meaning | +|------+------------------------------------| +| D | Draft message | +| F | Flagged message | +| P | Passed message (i.e., `forwarded') | +| R | Replied message | +| S | Seen message | +| T | Trashed; to be deleted later | + +New messages (in the ~new/~ sub-directory) do not have flags encoded in their +file-name; but we *mu* uses `N' in the ~--flags~ to represent that: + +#+ATTR_MAN: :disable-caption t +| Flag | Meaning | +|------+---------| +| N | New | + +Thus, changing flags means changing the letters at the end of the message +file-name, except when setting or removing the `N' (new) flag. Setting or +un-setting the New flag causes the message is to be moved from ~cur/~ to ~new/~ or +vice-versa, respectively. When marking a message as New, it looses the other +flags. + +* ABSOLUTE AND RELATIVE FLAGS + +You can specify the flags with the ~--flags~ parameter, and do either with either +*absolute* or *relative* flags. + +Absolute flags just specify the new flags by their letters; e.g. to specify a +/Trashed/, /Seen/, /Replied/ message, you'd use ~--flags STR~. +#+end_example + +Relative flags are relative to the current flags for some message, and each of +the flags is prefixed with either ~+~ ("add this flag") or ~-~ ("remove this flag"). + +So to add the /Seen/ flag and remove the /Draft/ flag from whatever the message +already has, ~--flags +S-D~. + +You cannot combine relative and relative flags. + +* EXAMPLES + +** change some flags +#+begin_example +$ mu move /home/user/Maildir/inbox/cur/1695559560.a73985881f4611ac2.hostname!2,S --flags +F +/home/user/Maildir/inbox/cur/1695559560.a73985881f4611ac2.hostname!2,FS +#+end_example + +** move to a different maildir +#+begin_example +$ mu move /home/user/Maildir/project1/cur/1695559560.a73985881f4611ac2.hostname!2,S /project2 +/home/user/Maildir/project2/cur/1695559560.a73985881f4611ac2.hostname!2,S +#+end_example + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*maildir(5)* diff --git a/man/mu-query.7.org b/man/mu-query.7.org new file mode 100644 index 0000000..49b8c46 --- /dev/null +++ b/man/mu-query.7.org @@ -0,0 +1,384 @@ +#+TITLE: MU QUERY +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-query - a language for finding messages in *mu* databases. + +* DESCRIPTION + +The mu query language is the language used by *mu find* and *mu4e* to find messages +in *mu*'s Xapian database. The language is quite similar to Xapian's default +query-parser, but is an independent implementation that is customized for the +mu/mu4e use-case. + +Here, we give a structured but informal overview of the query language and +provide examples. As a companion to this, we recommend the *mu fields* and *mu +flags* commands to get an up-to-date list of the available fields and flags. + +Furthermore, *mu find* provides the ~--analyze~ option, which shows how *mu* +interprets your query; see the *ANALYZING QUERIES* section below. + +*NOTE:* if you use queries on the command-line (say, for *mu find*), you need to +quote any characters that would otherwise be interpreted by the shell, such as +*""*, *(* and *)* and whitespace. + +* TERMS + +The basic building blocks of a query are *terms*; these are just normal words like +`banana' or `hello', or words prefixed with a field-name which makes them apply +to just that field. See *mu info fields* for all the available fields. + +Some example queries: +#+begin_example +vacation +subject:capybara +maildir:/inbox +#+end_example + +Terms without an explicit field-prefix, (like `vacation' above) are interpreted +like: +#+begin_example +to:vacation or subject:vacation or body:vacation or ... +#+end_example + +The language is case-insensitive for terms and attempts to `flatten' diacritics, +so =angtrom= matches =Ångström=. + +If terms contain whitespace, they need to be quoted: +#+begin_example +subject:"hi there" +#+end_example +This is a so-called =phrase query=, which means that we match against subjects +that contain the literal phrase "hi there". Phrase queries only work for fields +that are /indexed/, i.e., fields with *index* in the *mu info fields* search column. + +Remember that you need to escape those quotes when using this from the +command-line: +#+begin_example +mu find subject:\\"hi there\\" +#+end_example + +* LOGICAL OPERATORS + +We can combine terms with logical operators -- binary ones: *and*, *or*, *xor* and the +unary *not*, with the conventional rules for precedence and association. The +operators are case-insensitive. + +You can also group things with *(* and *)*, so you can write: +#+begin_example +(subject:beethoven or subject:bach) and not body:elvis +#+end_example + +If you do not explicitly specify an operator between terms, *and* is implied, so +the queries +#+begin_example +subject:chip subject:dale +#+end_example +#+begin_example +subject:chip AND subject:dale +#+end_example +are equivalent. For readability, we recommend the second version. + +Note that a =pure not= - e.g. searching for *not apples* is quite a `heavy' query. + +* REGULAR EXPRESSIONS AND WILDCARDS + +The language supports matching basic PCRE regular expressions, see *pcre(3)*. + +Regular expressions are enclosed in *//*. Some examples: + +#+begin_example +subject:/h.llo/ # match hallo, hello, ... +subject:/ +#+end_example + +Note the difference between `maildir:/foo' and `maildir:/foo/'; the former +matches messages in the `/foo' maildir, while the latter matches all messages in +all maildirs that match `foo', such as `/foo', `/bar/cuux/foo', `/fooishbar' +etc. + +Wildcards are another mechanism for matching where a term with a rightmost *** +(and =only= in that position) matches any term that starts with the part before +the ***; they are therefore less powerful than regular expressions, but also much +faster: +#+begin_example +foo* +#+end_example +is equivalent to +#+begin_example +/foo.*/ +#+end_example + +Regular expressions can be useful, but are relatively slow. + +* FIELDS + +We already saw a number of search fields, such as *subject:* and *body:*. For the +full table with all details, including single-char shortcuts, try the command: +~mu info fields~. + +#+ATTR_MAN: :disable-caption t +#+begin_example ++-----------+----------+----------+-----------------------------+ +| flag | shortcut | category | description | ++-----------+----------+----------+-----------------------------+ +| draft | D | file | Draft (in progress) | ++-----------+----------+----------+-----------------------------+ +| flagged | F | file | User-flagged | ++-----------+----------+----------+-----------------------------+ +| passed | P | file | Forwarded message | ++-----------+----------+----------+-----------------------------+ +| replied | R | file | Replied-to | ++-----------+----------+----------+-----------------------------+ +| seen | S | file | Viewed at least once | ++-----------+----------+----------+-----------------------------+ +| trashed | T | file | Marked for deletion | ++-----------+----------+----------+-----------------------------+ +| new | N | maildir | New message | ++-----------+----------+----------+-----------------------------+ +| signed | z | content | Cryptographically signed | ++-----------+----------+----------+-----------------------------+ +| encrypted | x | content | Encrypted | ++-----------+----------+----------+-----------------------------+ +| attach | a | content | Has at least one attachment | ++-----------+----------+----------+-----------------------------+ +| unread | u | pseudo | New or not seen message | ++-----------+----------+----------+-----------------------------+ +| list | l | content | Mailing list message | ++-----------+----------+----------+-----------------------------+ +| personal | q | content | Personal message | ++-----------+----------+----------+-----------------------------+ +| calendar | c | content | Calendar invitation | ++-----------+----------+----------+-----------------------------+ +#+end_example + +(*) The language code for the text-body if found. This works only if ~mu~ was +built with CLD2 support. + +There are also the special fields *contact:*, which matches all contact-fields +(=from=, =to=, =cc= and =bcc=), and *recip*, which matches all recipient-fields (=to=, =cc= +and =bcc=). + +Hence, for instance, +#+begin_example +contact:fnorb@example.com +#+end_example +is equivalent to +#+begin_example +(from:fnorb@example.com or to:fnorb@example.com or + cc:from:fnorb@example.com or bcc:fnorb@example.com) +#+end_example + +* DATE RANGES + +The *date:* field takes a date-range, expressed as the lower and upper bound, +separated by *..*. Either lower or upper (but not both) can be omitted to create +an open range. + +Dates are expressed in local time and using ISO-8601 format (YYYY-MM-DD +HH:MM:SS); you can leave out the right part and *mu* adds the rest, depending on +whether this is the beginning or end of the range (e.g., as a lower bound, +`2015' would be interpreted as the start of that year; as an upper bound as the +end of the year). + +You can use `/' , `.', `-', `:' and `T' to make dates more human-readable. + +Some examples: +#+begin_example +date:20170505..20170602 +date:2017-05-05..2017-06-02 +date:..2017-10-01T12:00 +date:2015-06-01.. +date:2016..2016 +#+end_example + +You can also use the special `dates' *now* and *today*: +#+begin_example +date:20170505..now +date:today.. +#+end_example + +Finally, you can use relative `ago' times which express some time before now and +consist of a number followed by a unit, with units *s* for seconds, *M* for minutes, +*h* for hours, *d* for days, *w* for week, *m* for months and *y* for years. Some +examples: + +#+begin_example +date:3m.. +date:2017.01.01..5w +#+end_example + +* SIZE RANGES + +The *size* or *z* field allows you to match =size ranges= -- that is, match messages +that have a byte-size within a certain range. Units (b (for bytes), K (for 1000 +bytes) and M (for 1000 * 1000 bytes) are supported). Some examples: + +#+begin_example +size:10k..2m +size:10m.. +#+end_example + +* FLAG FIELD + +The *flag/g* field allows you to match message flags. The following fields are +available: +#+begin_example ++-----------+----------+----------+-----------------------------+ +| flag | shortcut | category | description | ++-----------+----------+----------+-----------------------------+ +| draft | D | file | Draft (in progress) | ++-----------+----------+----------+-----------------------------+ +| flagged | F | file | User-flagged | ++-----------+----------+----------+-----------------------------+ +| passed | P | file | Forwarded message | ++-----------+----------+----------+-----------------------------+ +| replied | R | file | Replied-to | ++-----------+----------+----------+-----------------------------+ +| seen | S | file | Viewed at least once | ++-----------+----------+----------+-----------------------------+ +| trashed | T | file | Marked for deletion | ++-----------+----------+----------+-----------------------------+ +| new | N | maildir | New message | ++-----------+----------+----------+-----------------------------+ +| signed | z | content | Cryptographically signed | ++-----------+----------+----------+-----------------------------+ +| encrypted | x | content | Encrypted | ++-----------+----------+----------+-----------------------------+ +| attach | a | content | Has at least one attachment | ++-----------+----------+----------+-----------------------------+ +| unread | u | pseudo | New or not seen message | ++-----------+----------+----------+-----------------------------+ +| list | l | content | Mailing list message | ++-----------+----------+----------+-----------------------------+ +| personal | q | content | Personal message | ++-----------+----------+----------+-----------------------------+ +| calendar | c | content | Calendar invitation | ++-----------+----------+----------+-----------------------------+ +#+end_example + +Some examples: +#+begin_example +flag:attach +flag:replied +g:x +#+end_example + +Encrypted messages may be signed as well, but this is only visible after +decrypting and thus invisible to *mu*. + +* PRIORITY FIELD + +The message priority field (*prio:*) has three possible values: *low*, *normal* or +*high*. For instance, to match high-priority messages: +#+begin_example +prio:high +#+end_example + +* MAILDIR + +The Maildir field describes the directory path starting *after* the Maildir root +directory, and before the =/cur/= or =/new/= part. So, for example, if there's a +message with the file name =~/Maildir/lists/running/cur/1234.213:2,=, you could +find it (and all the other messages in that same maildir) with: +#+begin_example +maildir:/lists/running +#+end_example + +Note the starting `/'. If you want to match mails in the `root' maildir, you can +do with a single `/': +#+begin_example +maildir:/ +#+end_example + +If you have maildirs (or any fields) that include spaces, you need to quote +them, ie. +#+begin_example +maildir:"/Sent Items" +#+end_example + +And once again, note that when using the command-line, such queries must be +quoted: +#+begin_example +mu find 'maildir:"/Sent Items"' +#+end_example + +Also note that you should *not* end the maildir with a ~/~, or it can be +misinterpreted as a regular expression term; see aforementioned. + +* MORE EXAMPLES + +Here are some simple examples of *mu* queries; you can make many more complicated +queries using various logical operators, parentheses and so on, but in the +author's experience, it's usually faster to find a message with a simple query +just searching for some words. + +Find all messages with both `bee' and `bird' (in any field) +#+begin_example +bee AND bird +#+end_example + +Find all messages with either Frodo or Sam: +#+begin_example +Frodo OR Sam +#+end_example + +Find all messages with the `wombat' as subject, and `capybara' anywhere: +#+begin_example +subject:wombat and capybara +#+end_example + +Find all messages in the `Archive' folder from Fred: +#+begin_example +from:fred and maildir:/Archive +#+end_example + +Find all unread messages with attachments: +#+begin_example +flag:attach and flag:unread +#+end_example + +Find all messages with PDF-attachments: +#+begin_example +mime:application/pdf +#+end_example + +Find all messages with attached images: +#+begin_example +mime:image/* +#+end_example + +Find all messages written in Dutch or German with the word `hallo': +#+begin_example +hallo and (lang:nl or lang:de) +#+end_example + +This is only available if your *mu* has support for this; see *mu info* and check +for "cld2-support*. + +* ANALZYING QUERIES + +Despite all the excellent documentation, in some cases it can be non-obvious how +~mu~ interprets your query. For that, you can ask ~mu~ to analyze the query -- that +is, show how ~mu~ interprets the query. + +This uses the the ~--analyze~ option to *mu find*. +#+begin_example +$ mu find subject:wombat AND date:3m.. size:..2000 --analyze +,* query: + subject:wombat AND date:3m.. size:..2000 +,* parsed query: + (and (subject "wombat") (date (range "2023-05-30T06:10:09Z" "")) (size (range "" "2000"))) +,* Xapian query: + Query((Swombat AND VALUE_GE 4 n64759341 AND VALUE_LE 17 i7d0)) +#+end_example + +The ~parsed query~ is usually the most useful one for understanding how *mu* +interprets your query. + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu-find(1)*, *mu-info(1), *pcre(3)* diff --git a/man/mu-remove.1.org b/man/mu-remove.1.org new file mode 100644 index 0000000..ff2c24c --- /dev/null +++ b/man/mu-remove.1.org @@ -0,0 +1,27 @@ +#+TITLE: MU REMOVE +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-remove - remove messages from the database. + +* SYNOPSIS + +*mu [common-options] remove [options] <file> [<files>]* + +* DESCRIPTION + +*mu remove* removes specific messages from the database, each of them specified by +their filename. The files do not have to exist in the file system. + +* REMOVE OPTIONS + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)*, *mu-index(1)*, *mu-add(1)* diff --git a/man/mu-server.1.org b/man/mu-server.1.org new file mode 100644 index 0000000..1814860 --- /dev/null +++ b/man/mu-server.1.org @@ -0,0 +1,91 @@ +#+TITLE: MU-SERVER +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-server - the mu backend for the mu4e e-mail client + +* SYNOPSIS + +mu [common-options] server + +* DESCRIPTION + +*mu server* starts a simple shell in which one can query and manipulate the mu +database. The output uses s-expressions. *mu server* is not meant for use by +humans, except for debugging purposes. Instead, it is designed specifically for +the *mu4e* e-mail client. + +#+begin_example + (<command-name> :param1 value1 :param2 value2) +#+end_example + +For example, to view a certain message, the command would be: + +#+begin_example + (view :docid 12345) +#+end_example + +Parameters can be sent in any order; they must be of the correct type though. +See *lib/utils/mu-sexp-parser.hh* and *lib/utils/mu-sexp-parser.cc* in source-tree +for the details. + +* OUTPUT FORMAT + +*mu server* accepts a number of commands, and delivers its results in the form: + +#+begin_example + \\376<length>\\377<s-expr> +#+end_example + +\\376 (one byte 0xfe), followed by the length of the s-expression expressed as +an hexadecimal number, followed by another \\377 (one byte 0xff), followed by +the actual s-expression. + +By prefixing the expression with its length, it can be processed more +efficiently. The \\376 and \\377 were chosen since they never occur in valid +UTF-8 (in which the s-expressions are encoded). + +* SERVER OPTIONS + +** --commands + +List available commands (and try with ~--verbose~) + +** --eval <expression> + +Evaluate a mu4e server s-expression + +** --allow-temp-file + +If set, allow for the output of some commands to use temp-files rather than +directly through the emacs process input/output. This is noticeably faster for +commands with a lot of output, esp. when the the temp-file uses a in-memory +file-system. + +* PERFORMANCE + +As an indication for the relative performance, we can simulate something ~mu4e~ +does; we take overall time of 50 such requests: + +#+begin_src sh +time build/mu/mu server --allow-temp-file --eval '(find :query "\"\"" :include-related t :threads t :maxnum 50000)' >/dev/null +#+end_src +(and ~--allow-temp-file~ for 1.11) + +#+ATTR_MAN: :disable-caption t +| release | time (sec) | +|---------------+------------| +| 1.8 | 8.6s | +| 1.10 | 5.7s | +| 1.11 (master) | 2.8s | + + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO +*mu(1)* diff --git a/man/mu-verify.1.org b/man/mu-verify.1.org new file mode 100644 index 0000000..9cc0933 --- /dev/null +++ b/man/mu-verify.1.org @@ -0,0 +1,55 @@ +#+TITLE: MU VERIFY +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-verify - verify message signatures and display information about them + +* SYNOPSIS + +*mu [common-options] verify [options] [<file> ... ]* + +* DESCRIPTION + +*mu verify* is the *mu* command for verifying message signatures (such as PGP/GPG +signatures) and displaying information about them. The sub-command works on +message files, and does not require the message to be indexed in the database. + +If no message file is provided, the command expects the message on +standard-input. + +* VERIFY OPTIONS + +** -r, --auto-retrieve +attempt to find keys online (see the *auto-key-retrieve* option in the *gnupg(1)* +documentation). + +** decrypt +attempt to decrypt the message + +#+include: "common-options.inc" :minlevel 1 + +* EXAMPLES + +To display aggregated (one-line) information about the verification status in a +message: +#+begin_example +$ mu verify msgfile +#+end_example + +To display information about all the signatures: +#+begin_example +$ mu verify --verbose msgfile +#+end_example + +If you only want to use the exit code, you can use: +#+begin_example +$ mu verify --quiet msgfile +#+end_example +which does not give any output unless there is an error. + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)* diff --git a/man/mu-view.1.org b/man/mu-view.1.org new file mode 100644 index 0000000..17351f7 --- /dev/null +++ b/man/mu-view.1.org @@ -0,0 +1,54 @@ +#+TITLE: MU VIEW +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-view - display an e-mail message file + +* SYNOPSIS + +mu [common options] view [options] [<file> ...] + +* DESCRIPTION + +*mu view* is the *mu* command for displaying e-mail message files. It works on +message files and does =not= require the message to be indexed in the database. + +The command shows some common headers (From:, To:, Cc:, Bcc:, Subject: and +Date:), the list of attachments and either the plain-text or html body of the +message (if any), or its s-expression representation. + +If no message file is provided, the command reads the message from +standard-input. + +* VIEW OPTIONS + +** --format,-o = <format> +use the given output format, one of: + +- ~plain~ - use the plain-text body; this is the default +- ~html~ - use the HTML body +- ~sexp~ - show the S-expression representation of the message + +** --summary-len=<number> +instead of displaying the full message, output a summary based upon the first +=<number>= lines of the message. + +** --terminate +terminate messages with \\​f (=form-feed=) characters when displaying them. This is +useful when you want to further process them. + +** --decrypt +attempt to decrypt encrypted message bodies. This is only possible if *mu* +was built with crypto-support. + +** --auto-retrieve +attempt to retrieve crypto-keys automatically from the network, when needed. + +#+include: "common-options.inc" :minlevel 1 + +* BUGS + +* SEE ALSO + +*mu(1)* diff --git a/man/mu.1.org b/man/mu.1.org new file mode 100644 index 0000000..026fd32 --- /dev/null +++ b/man/mu.1.org @@ -0,0 +1,90 @@ +#+TITLE: MU +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu - a set of tools to deal with Maildirs and message files, in particular to +index and search e-mail messages. + +* SYNOPSIS + +~mu~ [COMMON-OPTIONS] [[COMMAND] [COMMAND-OPTIONS]] + +For information about the common options, see *COMMON OPTIONS*. + +* DESCRIPTION + +~mu~ is the general command shows help about the specific commands: + +- ~add~: add specific messages to the database. +- ~cfind~: find contacts +- ~extract~: extract attachments and other MIME-parts +- ~find~: find messages in the database +- ~help~: get help for some command +- ~index~: (re)index the messages in a Maildir +- ~info~: show information about the mu database +- ~init~: initialize the mu database +- ~mkdir~: create a new Maildir +- ~remove~: remove specific messages from the database +- ~server~: start a server process (for ~mu4e~-internal use) +- ~view~: view a specific message + +Each of the commands have their own manpage ~mu-<command~>~. + +~mu~ is a set of tools for dealing with Maildirs and the e-mail messages +in them. + +~mu~'s main purpose is to enable searching of e-mail messages. It +does so by periodically scanning a Maildir directory tree and +analyzing the e-mail messages found (this is called `indexing'). The +results of this analysis are stored in a database, which can then be +queried. + +In addition to indexing and searching, ~mu~ also offers +functionality for viewing messages, extracting attachments and +creating maildirs, and searching and exporting contact information. + +~mu~ can be used from the command line or can be integrated with various +e-mail clients. + +This manpage gives a general overview of the available commands +(~index~, ~find~, etc.); each ~mu~ command has its own +man-page as well. + +* COLORS + +Some ~mu~ commands support colorized output, and do so by default. If you don't +want colors, you can use ~--nocolor~. + +* ENCODING + +~mu~'s output is in the current locale, with the exceptions of the output +specifically meant for output to UTF8-encoded files. In practice, this means +that the output of commands ~index~, ~view~, ~extract~ is always encoded according to +the current locale. + +The same is true for ~find~ and ~cfind~, with some exceptions, where +the output is always UTF-8, regardless of the locale: + +- For ~cfind~ the exception is ~--format=bbdb~. This is hard-coded to UTF-8, and as + such specified in the output-file, so emacs/bbdb can handle it correctly + without guessing. +- For ~find~ the output is encoded according the locale for ~--format=plain~ (the + default), and UTF-8 for all other formats. + +* DATABASE AND FILE + +Commands ~mu index~ and ~find~ and ~cfind~ work with the database, while the other +ones work on individual mail files. Hence, running ~view~, ~mkdir~ and ~extract~ does +not require the mu database. + +#+include: "common-options.inc" :minlevel 1 + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO +~mu-add(1)~, ~mu-cfind(1)~, ~mu-extract(1)~, ~mu-find(1)~, ~mu-help(1)~, ~mu-index(1)~, +~mu-info(1)~, ~mu-init(1)~, ~mu-mkdir(1)~, ~mu-remove(1)~, ~mu-server(1)~, ~mu-view(1)~, +~mu-query(7)~, ~mu-easy(1)~ diff --git a/man/muhome.inc b/man/muhome.inc new file mode 100644 index 0000000..8b312a2 --- /dev/null +++ b/man/muhome.inc @@ -0,0 +1,12 @@ +** --muhome +use a non-default directory to store and read the database, write the logs, etc. +By default, ~mu~ uses the XDG Base Directory Specification (e.g. on GNU/Linux this +defaults to =~/.cache/mu= and =~/.config/mu=). Earlier versions of ~mu~ defaulted to +=~/.mu=, which now requires =--muhome=~/.mu=. + +The environment variable ~MUHOME~ can be used as an alternative to ~--muhome~. The +latter has precedence. + +# Local Variables: +# mode: org +# End: diff --git a/man/prefooter.inc b/man/prefooter.inc new file mode 100644 index 0000000..79c6e40 --- /dev/null +++ b/man/prefooter.inc @@ -0,0 +1,9 @@ +#+include: "bugs.inc" :minlevel 1 + +#+include: "author.inc" :minlevel 1 + +#+include: "copyright.inc" :minlevel 1 + +# Local Variables: +# mode: org +# End: diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..718d68f --- /dev/null +++ b/meson.build @@ -0,0 +1,306 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +################################################################################ +# project setup +project('mu', ['c', 'cpp'], + version: '1.12.5', + meson_version: '>= 0.56.0', + license: 'GPL-3.0-or-later', + default_options : [ + 'buildtype=debugoptimized', + 'warning_level=3', + 'c_std=c11', + 'cpp_std=c++17']) + +# hard-code the date here (for reproduciblity); we derive the dates used in e.g. +# documentation from this. +mu_date='2024-04-15' + +# installation paths +prefixdir = get_option('prefix') +bindir = prefixdir / get_option('bindir') +datadir = prefixdir / get_option('datadir') +mandir = prefixdir / get_option('mandir') +infodir = prefixdir / get_option('infodir') + +# allow for configuring lispdir, as with autotools. +if get_option('lispdir') == '' + mu4e_lispdir= datadir / join_paths('emacs', 'site-lisp', 'mu4e') +else + mu4e_lispdir= get_option('lispdir') / 'mu4e' +endif + +################################################################################ +# compilers / flags +# + +# compilers +cc = meson.get_compiler('c') +cxx= meson.get_compiler('cpp') + +extra_flags = [ + '-Wno-unused-parameter', + '-Wno-cast-function-type', + '-Wformat-security', + '-Wformat=2', + '-Wstack-protector', + '-fstack-protector-strong', + '-Wno-switch-enum', + # assuming these are false alarm... (in fmt, with gcc13): + '-Wno-array-bounds', + '-Wno-stringop-overflow',] + +if (cxx.get_id() == 'clang') + extra_flags += [ + '-Wc11-extensions', + '-Wno-keyword-macro', + '-Wno-deprecated-volatile', + '-Wno-#warnings'] +endif + +extra_cpp_flags= [ + '-Wno-volatile' +] + +if get_option('buildtype') == 'debug' + extra_flags += [ + '-D_GLIBCXX_ASSERTIONS', + '-ggdb', + '-g3'] +endif + +# extra arguments, if available +foreach extra_arg : extra_flags + if cc.has_argument (extra_arg) + add_project_arguments([extra_arg], language: 'c') + endif +endforeach + +foreach extra_arg : extra_flags + extra_cpp_flags + if cxx.has_argument (extra_arg) + add_project_arguments([extra_arg], language: 'cpp') + endif +endforeach + +# some clang don't have charconv, but we need it. +# https://github.com/djcb/mu/issues/2347 +cxx.check_header('charconv', required:true) + + +build_aux = join_paths(meson.current_source_dir(), 'build-aux') +################################################################################ +# derived date values (based on 'mu-date'); used in docs +# we can't use the 'date' because MacOS 'date' is incompatible with GNU's. +pdate=find_program(join_paths(build_aux, 'date.py')) +env = environment() +env.set('LANG', 'C') +mu_day_month_year = run_command(pdate, mu_date, '%d %B %Y', + check:true, capture:true, + env: env).stdout().strip() +mu_month_year = run_command(pdate, mu_date, '%B %Y', + check:true, capture:true, + env: env).stdout().strip() +mu_year = run_command(pdate, mu_date, '%Y', + check:true, capture:true, env: env).stdout().strip() + +################################################################################ +# config.h setup +# +config_h_data=configuration_data() +config_h_data.set('MU_STORE_SCHEMA_VERSION', 500) +config_h_data.set_quoted('PACKAGE_VERSION', meson.project_version()) +config_h_data.set_quoted('PACKAGE_STRING', meson.project_name() + ' ' + + meson.project_version()) +config_h_data.set_quoted('VERSION', meson.project_version()) +config_h_data.set_quoted('PACKAGE_NAME', meson.project_name()) + +add_project_arguments(['-DHAVE_CONFIG_H'], language: 'c') +add_project_arguments(['-DHAVE_CONFIG_H'], language: 'cpp') +config_h_dep=declare_dependency( + include_directories: include_directories(['.'])) + + +# +# d_type, d_ino are not available universally, so let's check +# (we use them for optimizations in mu-scanner +# +if cxx.has_member('struct dirent', 'd_ino', prefix : '#include<dirent.h>') + config_h_data.set('HAVE_DIRENT_D_INO', 1) +endif + +if cxx.has_member('struct dirent', 'd_type', prefix : '#include<dirent.h>') + config_h_data.set('HAVE_DIRENT_D_TYPE', 1) +endif + + +functions=[ + 'setsid' +] +foreach f : functions + if cc.has_function(f) + define = 'HAVE_' + f.underscorify().to_upper() + config_h_data.set(define, 1) + endif +endforeach + +if cc.has_function('wordexp') + config_h_data.set('HAVE_WORDEXP_H',1) +else + message('no wordexp, no command-line option expansion') +endif + +if not get_option('tests').disabled() + # only needed for tests + cp=find_program('cp') + ln=find_program('ln') + rm=find_program('rm') + + config_h_data.set_quoted('CP_PROGRAM', cp.full_path()) + config_h_data.set_quoted('RM_PROGRAM', rm.full_path()) + config_h_data.set_quoted('LN_PROGRAM', ln.full_path()) + + testmaildir=join_paths(meson.current_source_dir(), 'testdata') + config_h_data.set_quoted('MU_TESTMAILDIR', join_paths(testmaildir, 'testdir')) + config_h_data.set_quoted('MU_TESTMAILDIR2', join_paths(testmaildir, 'testdir2')) + config_h_data.set_quoted('MU_TESTMAILDIR4', join_paths(testmaildir, 'testdir4')) + config_h_data.set_quoted('MU_TESTMAILDIR_CJK', join_paths(testmaildir, 'cjk')) +endif + + +################################################################################ +# hard dependencies +# +glib_dep = dependency('glib-2.0', version: '>= 2.60') +gobject_dep = dependency('gobject-2.0', version: '>= 2.60') +gio_dep = dependency('gio-2.0', version: '>= 2.60') +gio_unix_dep = dependency('gio-unix-2.0', version: '>= 2.60') +gmime_dep = dependency('gmime-3.0', version: '>= 3.2') +thread_dep = dependency('threads') + +# we need Xapian 1.4 +xapian_dep = dependency('xapian-core', version:'>= 1.4', required:true) +xapver = xapian_dep.version() +if xapver.version_compare('>= 1.4.6') + message('xapian ' + xapver + ' supports c++ move-semantics') + config_h_data.set('HAVE_XAPIAN_MOVE_SEMANTICS', 1) +endif +if xapver.version_compare('>= 1.4.23') + message('xapian ' + xapver + ' supports ngrams') + config_h_data.set('HAVE_XAPIAN_FLAG_NGRAMS', 1) +endif + +# optionally, use Compact Language Detector2 if we can find it. +cld2_dep = meson.get_compiler('cpp').find_library('cld2', required: get_option('cld2')) +if not get_option('cld2').disabled() and cld2_dep.found() + config_h_data.set('HAVE_CLD2', 1) +else + message('CLD2 not found or disabled; no support for language detection') +endif + +# soft dependencies +guile_dep = dependency('guile-3.0', required: get_option('guile')) +# allow for a custom guile-extension-dir +if guile_dep.found() + custom_guile_xd=get_option('guile-extension-dir') + if custom_guile_xd == '' + guile_extension_dir = guile_dep.get_variable(pkgconfig: 'extensiondir') + else + guile_extension_dir = custom_guile_xd + endif + config_h_data.set_quoted('MU_GUILE_EXTENSION_DIR', guile_extension_dir) + message('Using guile-extension-dir: ' + guile_extension_dir) +endif + +makeinfo=find_program(['makeinfo'], required:false) +if not makeinfo.found() + message('makeinfo (texinfo) not found; not building info documentation') +else + install_info=find_program(['install-info'], required:false) + if not install_info.found() + message('install-info not found') + else + install_info_script=join_paths(build_aux, 'meson-install-info.sh') + endif +endif + +# readline. annoyingly, macos has an incompatible libedit claiming to be +# readline. this is only a dev/debug convenience for the mu4e repl. +readline_dep=[] +if get_option('readline').enabled() + readline_dep = dependency('readline', version:'>= 8.0') + config_h_data.set('HAVE_LIBREADLINE', 1) + config_h_data.set('HAVE_READLINE_READLINE_H', 1) + config_h_data.set('HAVE_READLINE_HISTORY', 1) + config_h_data.set('HAVE_READLINE_HISTORY_H', 1) +endif + + +################################################################################ +# write out version.texi (for texinfo builds in mu4e, guile) +version_texi_data=configuration_data() +version_texi_data.set('VERSION', meson.project_version()) +version_texi_data.set('EDITION', meson.project_version()) + +# derived date values +version_texi_data.set('UPDATED', mu_day_month_year) +version_texi_data.set('UPDATEDMONTH', mu_month_year) +version_texi_data.set('UPDATEDYEAR', mu_year) + +configure_file(input: join_paths(build_aux, 'version.texi.in'), + output: 'version.texi', + configuration: version_texi_data) + +################################################################################ +# install some data files +install_data('NEWS.org', install_dir : join_paths(datadir,'doc', 'mu')) + +################################################################################ +# subdirs +subdir('lib') +subdir('mu') + + +# emacs -- needed for mu4e compilation +emacs_name=get_option('emacs') +emacs_min_version='26.3' +emacs=find_program([emacs_name], version: '>='+emacs_min_version, required:false) +if emacs.found() + subdir('man') + subdir('mu4e') +else + message('emacs not found; not pre-compiling mu4e / generating manpages') +endif + +if not get_option('guile').disabled() and guile_dep.found() + config_h_data.set('BUILD_GUILE', 1) + config_h_data.set_quoted('GUILE_BINARY', + guile_dep.get_variable(pkgconfig: 'guile')) + #message('guile is disabled for now') + subdir('guile') +endif + +config_h_data.set_quoted('MU_PROGRAM', mu.full_path()) +################################################################################ + +################################################################################ +# write-out config.h +configure_file(output : 'config.h', configuration : config_h_data) + +if gmime_dep.version() == '3.2.13' + warning('gmime version 3.2.13 detected, which as a decoding bug') + warning('See: https://github.com/jstedfast/gmime/issues/133') +endif diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 0000000..93bc7db --- /dev/null +++ b/meson_options.txt @@ -0,0 +1,49 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +option('tests', + type : 'feature', + value: 'auto', + description: 'build unit tests') + +option('guile', + type : 'feature', + value: 'auto', + description: 'build the guile scripting support (requires guile-3.x)') + +option('cld2', + type : 'feature', + value: 'auto', + description: 'Compact Language Detector2') + +# by default, this uses guile_dep.get_variable(pkgconfig: 'extensiondir') +option('guile-extension-dir', + type: 'string', + description: 'custom install path for the guile extension module') + +option('readline', + type: 'feature', + value: 'auto', + description: 'enable readline support for the mu4e repl') + +option('emacs', + type: 'string', + value: 'emacs', + description: 'name/path of the emacs executable') + +option('lispdir', + type: 'string', + description: 'path under which to install emacs-lisp files') diff --git a/mu/meson.build b/mu/meson.build new file mode 100644 index 0000000..0ada1e5 --- /dev/null +++ b/mu/meson.build @@ -0,0 +1,43 @@ +## Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +mu = executable( + 'mu', [ + 'mu.cc', + 'mu-options.cc', + 'mu-cmd-add.cc', + 'mu-cmd-cfind.cc', + 'mu-cmd-extract.cc', + 'mu-cmd-find.cc', + 'mu-cmd-info.cc', + 'mu-cmd-init.cc', + 'mu-cmd-index.cc', + 'mu-cmd-mkdir.cc', + 'mu-cmd-move.cc', + 'mu-cmd-remove.cc', + 'mu-cmd-script.cc', + 'mu-cmd-server.cc', + 'mu-cmd-verify.cc', + 'mu-cmd-view.cc', + 'mu-cmd.cc' +], + dependencies: [ glib_dep, gmime_dep, lib_mu_dep, thread_dep, config_h_dep ], + cpp_args: ['-DMU_SCRIPTS_DIR="'+ join_paths(datadir, 'mu', 'scripts') + '"'], + install: true) +# +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/mu/mu-cmd-add.cc b/mu/mu-cmd-add.cc new file mode 100644 index 0000000..46dcf9a --- /dev/null +++ b/mu/mu-cmd-add.cc @@ -0,0 +1,125 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +using namespace Mu; + +Result<void> +Mu::mu_cmd_add(Mu::Store& store, const Options& opts) +{ + for (auto&& file: opts.add.files) { + const auto docid{store.add_message(file)}; + if (!docid) + return Err(docid.error()); + else + mu_debug("added message @ {}, docid={}", file, *docid); + } + + return Ok(); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_add_ok() +{ + auto testhome{unwrap(make_temp_dir())}; + auto dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + + { + unwrap(Store::make_new(dbpath, MU_TESTMAILDIR)); + } + + { + auto res = run_command({MU_PROGRAM, "add", mu_format("--muhome={}", testhome), + MU_TESTMAILDIR "/cur/1220863042.12663_1.mindcrime!2,S"}); + assert_valid_command(res); + } + + { + auto&& store = Store::make(dbpath); + assert_valid_result(store); + g_assert_cmpuint(store->size(),==,1); + } + + { // re-add the same + auto res = run_command({MU_PROGRAM, "add", mu_format("--muhome={}",testhome), + MU_TESTMAILDIR "/cur/1220863042.12663_1.mindcrime!2,S"}); + assert_valid_command(res); + } + + { + auto&& store = Store::make(dbpath); + assert_valid_result(store); + g_assert_cmpuint(store->size(),==,1); + } + + + remove_directory(testhome); +} + +static void +test_add_fail() +{ + auto testhome{unwrap(make_temp_dir())}; + auto dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + + { + unwrap(Store::make_new(dbpath, MU_TESTMAILDIR2)); + } + + { // wrong maildir + auto res = run_command({MU_PROGRAM, "add", mu_format("--muhome={}", testhome), + MU_TESTMAILDIR "/cur/1220863042.12663_1.mindcrime!2,S"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,!=,0); + } + + + { // non-existent + auto res = run_command({MU_PROGRAM, "add", mu_format("--muhome={}", testhome), + "/foo/bar/non-existent"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,!=,0); + } + + remove_directory(testhome); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/add/ok", test_add_ok); + g_test_add_func("/cmd/add/fail", test_add_fail); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-cfind.cc b/mu/mu-cmd-cfind.cc new file mode 100644 index 0000000..9c61595 --- /dev/null +++ b/mu/mu-cmd-cfind.cc @@ -0,0 +1,535 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include <cstdint> +#include <string> +#include <functional> +#include <unordered_map> + +#include <utils/mu-utils.hh> +#include <utils/mu-regex.hh> +#include <utils/mu-option.hh> + +using namespace Mu; + +enum struct ItemType { Header, Footer, Normal }; +using OutputFunc = std::function<void(ItemType itype, Option<const Contact&>, const Options&)>; +using OptContact = Option<const Contact&>; +using Format = Options::Cfind::Format; + +// simplistic guess of first & last names, for setting +// some initial value. +static std::pair<std::string, std::string> +guess_first_last_name(const std::string& name) +{ + if (name.empty()) + return {}; + + const auto lastspc = name.find_last_of(' '); + if (lastspc == name.npos) + return { name, "" }; // no last name + else + return { name.substr(0, lastspc), name.substr(lastspc + 1)}; +} + + +// candidate nick and a _count_ for that given nick, to uniquify them. +static std::unordered_map<std::string, size_t> nicks; +static std::string +guess_nick(const Contact& contact) +{ + auto cleanup = [](const std::string& str) { + std::string clean; + for (auto& c: str) // XXX: support non-ascii + if (!::ispunct(c) && !::isspace(c)) + clean += c; + return clean; + }; + + auto nick = cleanup(std::invoke([&]()->std::string { + + // no name? use the user part from the addr + if (contact.name.empty()) { + const auto pos{contact.email.find('@')}; + if (pos == std::string::npos) + return contact.email; // no '@' + else + return contact.email.substr(0, pos); + } + + const auto names{guess_first_last_name(contact.name)}; + /* if there's no last name, use first name as the nick */ + if (names.second.empty()) + return names.first; + + char initial[7] = {}; + if (g_unichar_to_utf8(g_utf8_get_char(names.second.c_str()), initial) == 0) { + /* couldn't we get an initial for the last name? + * just use the first name*/ + return names.first; + } else // prepend the initial + return names.first + initial; + })); + + // uniquify. + if (auto it = nicks.find(nick); it == nicks.cend()) + nicks.emplace(nick, 0); + else { + ++it->second; + nick = mu_format("{}{}", nick, ++it->second); + } + + return nick; +} + + +static void +output_plain(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact) + return; + + const auto col1{opts.nocolor ? "" : MU_COLOR_MAGENTA}; + const auto col2{opts.nocolor ? "" : MU_COLOR_GREEN}; + const auto coldef{opts.nocolor ? "" : MU_COLOR_DEFAULT}; + + mu_print_encoded("{}{}{}{}{}{}{}\n", + col1, contact->name, coldef, + contact->name.empty() ? "" : " ", + col2, contact->email, coldef); +} + +static void +output_mutt_alias(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact) + return; + + const auto nick{guess_nick(*contact)}; + mu_print_encoded("alias {} {} <{}>\n", nick, contact->name, contact->email); + +} + +static void +output_mutt_address_book(ItemType itype, OptContact contact, const Options& opts) +{ + if (itype == ItemType::Header) + mu_print ("Matching addresses in the mu database:\n"); + + if (contact) + mu_print_encoded("{}\t{}\t\n", contact->email, contact->name); +} + +static void +output_wanderlust(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact || contact->name.empty()) + return; + + auto nick=guess_nick(*contact); + + mu_print_encoded("{} \"{}\" \"{}\"\n", contact->email, nick, contact->name); + +} + +static void +output_org_contact(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact || contact->name.empty()) + return; + + mu_print_encoded("* {}\n:PROPERTIES:\n:EMAIL: {}\n:END:\n\n", + contact->name, contact->email); +} + +static void +output_bbdb(ItemType itype, OptContact contact, const Options& opts) +{ + if (itype == ItemType::Header) + mu_println (";; -*-coding: utf-8-emacs;-*-\n" + ";;; file-version: 6"); + if (!contact) + return; + + const auto names{guess_first_last_name(contact->name)}; + const auto now{mu_format("{:%Y-%m-%d}", mu_time(::time({})))}; + const auto timestamp{mu_format("{:%Y-%m-%d}", mu_time(contact->message_date))}; + + mu_println("[\"{}\" \"{}\" nil nil nil nil (\"{}\") " + "((creation-date . \"{}\") (time-stamp . \"{}\")) nil]", + names.first, names.second, contact->email, now, timestamp); +} + +static void +output_csv(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact) + return; + + mu_print_encoded("{},{}\n", + contact->name.empty() ? "" : Mu::quote(contact->name), + Mu::quote(contact->email)); +} + +static void +output_json(ItemType itype, OptContact contact, const Options& opts) +{ + if (itype == ItemType::Header) + mu_println("["); + if (contact) { + mu_print("{}", itype == ItemType::Header ? "" : ",\n"); + mu_println (" {{"); + + const std::string name = contact->name.empty() ? "null" : Mu::quote(contact->name); + mu_print_encoded( + " \"email\" : \"{}\",\n" + " \"name\" : {},\n" + " \"display\" : {},\n" + " \"last-seen\" : {},\n" + " \"last-seen-iso\" : \"{}\",\n" + " \"personal\" : {},\n" + " \"frequency\" : {}\n", + contact->email, + name, + Mu::quote(contact->display_name()), + contact->message_date, + mu_format("{:%FT%TZ}", mu_time(contact->message_date, true/*utc*/)), + contact->personal ? "true" : "false", + contact->frequency); + mu_print(" }}"); + } + + if (itype == ItemType::Footer) + mu_println("\n]"); +} + +static OutputFunc +find_output_func(Format format) +{ +#pragma GCC diagnostic push +#pragma GCC diagnostic error "-Wswitch" + switch(format) { + case Format::Plain: + return output_plain; + case Format::MuttAlias: + return output_mutt_alias; + case Format::MuttAddressBook: + return output_mutt_address_book; + case Format::Wanderlust: + return output_wanderlust; + case Format::OrgContact: + return output_org_contact; + case Format::Bbdb: + return output_bbdb; + case Format::Csv: + return output_csv; + case Format::Json: + return output_json; + default: + mu_warning("unsupported format"); + return {}; + } +#pragma GCC diagnostic pop +} + + +Result<void> +Mu::mu_cmd_cfind(const Mu::Store& store, const Mu::Options& opts) +{ + size_t num{}; + OutputFunc output = find_output_func(opts.cfind.format); + if (!output) + return Err(Error::Code::Internal, + "missing output function"); + + // get the pattern regex, if any. + Regex rx{}; + if (!opts.cfind.rx_pattern.empty()) { + if (auto&& res = Regex::make(opts.cfind.rx_pattern, + static_cast<GRegexCompileFlags> + (G_REGEX_OPTIMIZE|G_REGEX_CASELESS)); !res) + return Err(std::move(res.error())); + else + rx = res.value(); + } + + nicks.clear(); + store.contacts_cache().for_each([&](const Contact& contact)->bool { + + if (opts.cfind.maxnum && num > *opts.cfind.maxnum) + return false; /* stop the loop */ + + if (!store.contacts_cache().is_valid(contact.email)) + return true; /* next */ + + // filter for maxnum, personal & "after" + if ((opts.cfind.personal && !contact.personal) || + (opts.cfind.after.value_or(0) > contact.message_date)) + return true; /* next */ + + // filter for regex, if any. + if (rx) { + if (!rx.matches(contact.name) && !rx.matches(contact.email)) + return true; /* next */ + } + + /* seems we have a match! display it. */ + const auto itype{num == 0 ? ItemType::Header : ItemType::Normal}; + output(itype, contact, opts); + ++num; + return true; + }); + + if (num == 0) + return Err(Error::Code::NoMatches, "no matching contacts found"); + + output(ItemType::Footer, Nothing, opts); + return Ok(); +} + + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + + +static std::string test_mu_home; + +static void +test_mu_cfind_plain(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "plain", "testmu\\.xxx?"})}; + assert_valid_result(res); + + /* note, output order is unspecified */ + if (res->standard_out[0] == 'H') + assert_equal(res->standard_out, + "Helmut Kröger hk@testmu.xxx\n" + "Mü testmu@testmu.xx\n"); + else + assert_equal(res->standard_out, + "Mü testmu@testmu.xx\n" + "Helmut Kröger hk@testmu.xxx\n"); +} + +static void +test_mu_cfind_bbdb(void) +{ + const auto old_tz{set_tz("Europe/Helsinki")}; + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "bbdb", "testmu\\.xxx?"})}; + assert_valid_result(res); + g_assert_cmpuint(res->standard_out.size(), >, 52); + +#define frm1 \ + ";; -*-coding: utf-8-emacs;-*-\n" \ + ";;; file-version: 6\n" \ + "[\"Helmut\" \"Kröger\" nil nil nil nil (\"hk@testmu.xxx\") " \ + "((creation-date . \"{}\") " \ + "(time-stamp . \"1970-01-01\")) nil]\n" \ + "[\"Mü\" \"\" nil nil nil nil (\"testmu@testmu.xx\") " \ + "((creation-date . \"{}\") " \ + "(time-stamp . \"1970-01-01\")) nil]\n" + +#define frm2 \ + ";; -*-coding: utf-8-emacs;-*-\n" \ + ";;; file-version: 6\n" \ + "[\"Mü\" \"\" nil nil nil nil (\"testmu@testmu.xx\") " \ + "((creation-date . \"{}\") " \ + "(time-stamp . \"1970-01-01\")) nil]\n" \ + "[\"Helmut\" \"Kröger\" nil nil nil nil (\"hk@testmu.xxx\") " \ + "((creation-date . \"{}\") " \ + "(time-stamp . \"1970-01-01\")) nil]\n" + + auto&& today{mu_format("{:%F}", mu_time(::time({})))}; + std::string expected; + if (res->standard_out.at(52) == 'H') + expected = mu_format(frm1, today, today); + else + expected = mu_format(frm2, today, today); + + assert_equal(res->standard_out, expected); + set_tz(old_tz); +} + +static void +test_mu_cfind_wl(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "wl", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(0) == 'h') + assert_equal(res->standard_out, + "hk@testmu.xxx \"HelmutK\" \"Helmut Kröger\"\n" + "testmu@testmu.xx \"Mü\" \"Mü\"\n"); + else + assert_equal(res->standard_out, + "testmu@testmu.xx \"Mü\" \"Mü\"\n" + "hk@testmu.xxx \"HelmutK\" \"Helmut Kröger\"\n"); +} + +static void +test_mu_cfind_mutt_alias(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "mutt-alias", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(6) == 'H') + assert_equal(res->standard_out, + "alias HelmutK Helmut Kröger <hk@testmu.xxx>\n" + "alias Mü Mü <testmu@testmu.xx>\n"); + else + assert_equal(res->standard_out, + "alias Mü Mü <testmu@testmu.xx>\n" + "alias HelmutK Helmut Kröger <hk@testmu.xxx>\n"); +} + +static void +test_mu_cfind_mutt_ab(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "mutt-ab", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(39) == 'h') + assert_equal(res->standard_out, + "Matching addresses in the mu database:\n" + "hk@testmu.xxx\tHelmut Kröger\t\n" + "testmu@testmu.xx\tMü\t\n"); + else + assert_equal(res->standard_out, + "Matching addresses in the mu database:\n" + "testmu@testmu.xx\tMü\t\n" + "hk@testmu.xxx\tHelmut Kröger\t\n"); +} + +static void +test_mu_cfind_org_contact(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "org-contact", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(2) == 'H') + assert_equal(res->standard_out, + "* Helmut Kröger\n" + ":PROPERTIES:\n" + ":EMAIL: hk@testmu.xxx\n" + ":END:\n\n" + "* Mü\n" + ":PROPERTIES:\n" + ":EMAIL: testmu@testmu.xx\n" + ":END:\n\n"); + else + assert_equal(res->standard_out, + "* Mü\n" + ":PROPERTIES:\n" + ":EMAIL: testmu@testmu.xx\n" + ":END:\n\n" + "* Helmut Kröger\n" + ":PROPERTIES:\n" + ":EMAIL: hk@testmu.xxx\n" + ":END:\n\n"); +} + +static void +test_mu_cfind_csv(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "csv", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(1) == 'H') + assert_equal(res->standard_out, + "\"Helmut Kröger\",\"hk@testmu.xxx\"\n" + "\"Mü\",\"testmu@testmu.xx\"\n"); + else + assert_equal(res->standard_out, + "\"Mü\",\"testmu@testmu.xx\"\n" + "\"Helmut Kröger\",\"hk@testmu.xxx\"\n"); +} + + +static void +test_mu_cfind_json() +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "json", "^a@example\\.com"})}; + assert_valid_result(res); + + const auto expected = R"([ + { + "email" : "a@example.com", + "name" : null, + "display" : "a@example.com", + "last-seen" : 1463331445, + "last-seen-iso" : "2016-05-15T16:57:25Z", + "personal" : false, + "frequency" : 1 + } +] +)"; + assert_equal(res->standard_out, expected); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + if (!set_en_us_utf8_locale()) + return 0; /* don't error out... */ + + TempDir temp_dir{}; + { + test_mu_home = temp_dir.path(); + + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", + "--muhome", test_mu_home, "--maildir" , MU_TESTMAILDIR}); + assert_valid_result(res1); + + auto res2 = run_command({MU_PROGRAM, "--quiet", "index", + "--muhome", test_mu_home}); + assert_valid_result(res2); + } + + g_test_add_func("/cmd/find/plain", test_mu_cfind_plain); + g_test_add_func("/cmd/find/bbdb", test_mu_cfind_bbdb); + g_test_add_func("/cmd/find/wl", test_mu_cfind_wl); + g_test_add_func("/cmd/find/mutt-alias", test_mu_cfind_mutt_alias); + g_test_add_func("/cmd/find/mutt-ab", test_mu_cfind_mutt_ab); + g_test_add_func("/cmd/find/org-contact", test_mu_cfind_org_contact); + g_test_add_func("/cmd/find/csv", test_mu_cfind_csv); + g_test_add_func("/cmd/find/json", test_mu_cfind_json); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-extract.cc b/mu/mu-cmd-extract.cc new file mode 100644 index 0000000..28793b3 --- /dev/null +++ b/mu/mu-cmd-extract.cc @@ -0,0 +1,306 @@ +/* +** Copyright (C) 2010-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-regex.hh" +#include <message/mu-message.hh> + +using namespace Mu; + +static Result<void> +save_part(const Message::Part& part, size_t idx, const Options& opts) +{ + const auto targetdir = std::invoke([&]{ + const auto tdir{opts.extract.targetdir}; + return tdir.empty() ? tdir : tdir + G_DIR_SEPARATOR_S; + }); + + /* 'uncooked' isn't really _raw_; it means only doing some _minimal_ + * cooking */ + const auto path{targetdir + + part.cooked_filename(opts.extract.uncooked) + .value_or(mu_format("part-{}", idx))}; + + if (auto&& res{part.to_file(path, opts.extract.overwrite)}; !res) + return Err(res.error()); + else if (opts.extract.play) + return play(path); + else + return Ok(); +} + +static Result<void> +save_parts(const Message& message, const std::string& filename_rx, + const Options& opts) +{ + size_t partnum{}, saved_num{}; + for (auto&& part: message.parts()) { + ++partnum; + // should we extract this part? + const auto do_extract = std::invoke([&]() { + + if (opts.extract.save_all) + return true; + else if (opts.extract.save_attachments && + part.looks_like_attachment()) + return true; + else if (seq_some(opts.extract.parts, + [&](auto&& num){return num==partnum;})) + return true; + else if (!filename_rx.empty() && part.raw_filename()) { + if (auto rx = Regex::make(filename_rx); !rx) + throw rx.error(); + else if (rx->matches(*part.raw_filename())) + return true; + } + return false; + }); + + if (!do_extract) + continue; + + if (auto res = save_part(part, partnum, opts); !res) + return res; + + ++saved_num; + } + + if (saved_num == 0) + return Err(Error::Code::File, + "no {} extracted from this message", + opts.extract.save_attachments ? "attachments" : "parts"); + else + return Ok(); +} + +#define color_maybe(C) \ + do { \ + if (color) \ + fputs((C), stdout); \ + } while (0) + +static void +show_part(const MessagePart& part, size_t index, bool color) +{ + /* index */ + mu_print(" {} ", index); + + /* filename */ + color_maybe(MU_COLOR_GREEN); + const auto fname{part.raw_filename()}; + fputs_encoded(fname.value_or("<none>"), stdout); + fputs_encoded(" ", stdout); + + /* content-type */ + color_maybe(MU_COLOR_BLUE); + const auto ctype{part.mime_type()}; + fputs_encoded(ctype.value_or("<none>"), stdout); + + /* /\* disposition *\/ */ + color_maybe(MU_COLOR_MAGENTA); + mu_print_encoded(" [{}]", part.is_attachment() ? "attachment" : "inline"); + /* size */ + if (part.size() > 0) { + color_maybe(MU_COLOR_CYAN); + mu_print(" ({} bytes)", part.size()); + } + + color_maybe(MU_COLOR_DEFAULT); + fputs("\n", stdout); +} + +static Mu::Result<void> +show_parts(const Message& message, const Options& opts) +{ + size_t index{}; + mu_println("MIME-parts in this message:"); + for (auto&& part: message.parts()) + show_part(part, ++index, !opts.nocolor); + + return Ok(); +} + +Mu::Result<void> +Mu::mu_cmd_extract(const Options& opts) +{ + auto message = std::invoke([&]()->Result<Message>{ + const auto mopts{message_options(opts.extract)}; + if (!opts.extract.message.empty()) + return Message::make_from_path(opts.extract.message, mopts); + + const auto msgtxt = read_from_stdin(); + if (!msgtxt) + return Err(msgtxt.error()); + else + return Message::make_from_text(*msgtxt, {}, mopts); + }); + + if (!message) + return Err(message.error()); + else if (opts.extract.parts.empty() && + !opts.extract.save_attachments && !opts.extract.save_all && + opts.extract.filename_rx.empty()) + return show_parts(*message, opts); /* show, don't save */ + + if (!check_dir(opts.extract.targetdir, false/*!readable*/, true/*writeable*/)) + return Err(Error::Code::File, + "target '{}' is not a writable directory", + opts.extract.targetdir); + + return save_parts(*message, opts.extract.filename_rx, opts); +} + + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include <glib.h> +#include <glib/gstdio.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include <utils/mu-regex.hh> +#include "utils/mu-test-utils.hh" + + +static gint64 +get_file_size(const std::string& path) +{ + int rv; + struct stat statbuf; + + mu_info("ppatj {}", path); + + rv = stat(path.c_str(), &statbuf); + if (rv != 0) { + mu_debug ("error: {}", g_strerror (errno)); + return -1; + } + + mu_debug("{} -> {} bytes", path, statbuf.st_size); + + return statbuf.st_size; +} + +static void +test_mu_extract_02(void) +{ + TempDir temp_dir{}; + auto res= run_command({ + MU_PROGRAM, "extract", "--save-attachments", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(get_file_size(join_paths(temp_dir.path(), "custer.jpg")), >=, 15955); + g_assert_cmpuint(get_file_size(join_paths(temp_dir.path(), "custer.jpg")), <=, 15960); + g_assert_cmpuint(get_file_size(join_paths(temp_dir.path(), "sittingbull.jpg")), ==, 17674); +} + +static void +test_mu_extract_03(void) +{ + TempDir temp_dir{}; + auto res= run_command({ + MU_PROGRAM, "extract", "--parts=3", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_true(g_access(join_paths(temp_dir.path(), "custer.jpg").c_str(), F_OK) == 0); + g_assert_false(g_access(join_paths(temp_dir.path(), "sittingbull.jpg").c_str(), F_OK) == 0); +} + +static void +test_mu_extract_overwrite(void) +{ + TempDir temp_dir{}; + auto res= run_command({ + MU_PROGRAM, "extract", "-a", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_true(g_access(join_paths(temp_dir.path(), "custer.jpg").c_str(), F_OK) == 0); + g_assert_true(g_access(join_paths(temp_dir.path(), "sittingbull.jpg").c_str(), F_OK) == 0); + + + /* now, it should fail, because we don't allow overwrites + * without --overwrite */ + auto res2 = run_command({ + MU_PROGRAM, "extract", "-a", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + + assert_valid_result(res2); + g_assert_false(res2->standard_err.empty()); + + + auto res3 = run_command({ + MU_PROGRAM, "extract", "-a", "--overwrite", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + + assert_valid_result(res3); + g_assert_true(res3->standard_err.empty()); +} + +static void +test_mu_extract_by_name(void) +{ + TempDir temp_dir{}; + auto res= run_command({ + MU_PROGRAM, "extract", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5"), + "sittingbull.jpg"}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_true(g_access(join_paths(temp_dir.path(), "sittingbull.jpg").c_str(), F_OK) == 0); + g_assert_false(g_access(join_paths(temp_dir.path(), "custer.jpg").c_str(), F_OK) == 0); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/extract/02", test_mu_extract_02); + g_test_add_func("/cmd/extract/03", test_mu_extract_03); + g_test_add_func("/cmd/extract/overwrite", test_mu_extract_overwrite); + g_test_add_func("/cmd/extract/by-name", test_mu_extract_by_name); + + return g_test_run(); +} + +#endif diff --git a/mu/mu-cmd-find.cc b/mu/mu-cmd-find.cc new file mode 100644 index 0000000..3853856 --- /dev/null +++ b/mu/mu-cmd-find.cc @@ -0,0 +1,733 @@ + /* +** Copyright (C) 2008-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <array> + +#include <unistd.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> +#include <stdlib.h> +#include <signal.h> +#include <sys/wait.h> + +#include "message/mu-message.hh" +#include "mu-maildir.hh" +#include "mu-query-match-deciders.hh" +#include "mu-query.hh" +#include "mu-query-macros.hh" +#include "mu-query-parser.hh" +#include "message/mu-message.hh" + +#include "utils/mu-option.hh" + +#include "mu-cmd.hh" +#include "utils/mu-utils.hh" + +using namespace Mu; + +using Format = Options::Find::Format; + +struct OutputInfo { + Xapian::docid docid{}; + bool header{}; + bool footer{}; + bool last{}; + Option<QueryMatch&> match_info; +}; + +constexpr auto FirstOutput{OutputInfo{0, true, false, {}, {}}}; +constexpr auto LastOutput{OutputInfo{0, false, true, {}, {}}}; + +using OutputFunc = std::function<Result<void>(const Option<Message>& msg, const OutputInfo&, + const Options&)>; + +using Format = Options::Find::Format; + +static Result<void> +analyze_query_expr(const Store& store, const std::string& expr, const Options& opts) +{ + auto print_item=[&](auto&&title, auto&&val) { + const auto blue{opts.nocolor ? "" : MU_COLOR_BLUE}; + const auto green{opts.nocolor ? "" : MU_COLOR_GREEN}; + const auto reset{opts.nocolor ? "" : MU_COLOR_DEFAULT}; + mu_println("* {}{}{}:\n {}{}{}", blue, title, reset, green, val, reset); + }; + + print_item("query", expr); + + const auto pq{parse_query(expr, false/*don't expand*/).to_string()}; + const auto pqx{parse_query(expr, true/*do expand*/).to_string()}; + + print_item("parsed query", pq); + if (pq != pqx) + print_item("parsed query (expanded)", pqx); + + auto xq{make_xapian_query(store, expr)}; + if (!xq) + return Err(std::move(xq.error())); + + print_item("Xapian query", xq->get_description()); + + return Ok(); +} + +static Result<QueryResults> +run_query(const Store& store, const std::string& expr, const Options& opts) +{ + Mu::QueryFlags qflags{QueryFlags::SkipUnreadable}; + if (opts.find.reverse) + qflags |= QueryFlags::Descending; + if (opts.find.skip_dups) + qflags |= QueryFlags::SkipDuplicates; + if (opts.find.include_related) + qflags |= QueryFlags::IncludeRelated; + if (opts.find.threads) + qflags |= QueryFlags::Threading; + + return store.run_query(expr, + opts.find.sortfield, + qflags, opts.find.maxnum.value_or(0)); +} + +static Result<void> +exec_cmd(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (!msg) + return Ok(); + + int wait_status{}; + GError *err{}; + auto cmdline{mu_format("{} {}", opts.find.exec, + to_string_gchar(g_shell_quote(msg->path().c_str())))}; + + if (!g_spawn_command_line_sync(cmdline.c_str(), {}, {}, &wait_status, &err)) + return Err(Error::Code::File, &err/*consumed*/, + "failed to execute shell command"); + else if (WEXITSTATUS(wait_status) != 0) + return Err(Error::Code::File, + "shell command exited with exit-code {}", + WEXITSTATUS(wait_status)); + return Ok(); +} + +static Result<std::string> +resolve_bookmark(const Store& store, const Options& opts) +{ + QueryMacros macros{store.config()}; + if (auto&& res{macros.load_bookmarks(opts.runtime_path(RuntimePath::Bookmarks))}; !res) + return Err(res.error()); + else if (auto&& bm{macros.find_macro(opts.find.bookmark)}; !bm) + return Err(Error::Code::InvalidArgument, "bookmark '{}' not found", + opts.find.bookmark); + else + return Ok(std::move(*bm)); +} + +static Result<std::string> +get_query(const Store& store, const Options& opts) +{ + if (opts.find.bookmark.empty() && opts.find.query.empty()) + return Err(Error::Code::InvalidArgument, + "neither bookmark nor query"); + + std::string bookmark; + if (!opts.find.bookmark.empty()) { + const auto res = resolve_bookmark(store, opts); + if (!res) + return Err(std::move(res.error())); + bookmark = res.value() + " "; + } + + auto&& query{join(opts.find.query, " ")}; + return Ok(bookmark + query); +} + +static Result<void> +prepare_links(const Options& opts) +{ + /* note, mu_maildir_mkdir simply ignores whatever part of the + * mail dir already exists */ + if (auto&& res = maildir_mkdir(opts.find.linksdir, 0700, true); !res) + return Err(std::move(res.error())); + + if (!opts.find.clearlinks) + return Ok(); + + if (auto&& res = maildir_clear_links(opts.find.linksdir); !res) + return Err(std::move(res.error())); + + return Ok(); +} + +static Result<void> +output_link(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (info.header) + return prepare_links(opts); + else if (info.footer) + return Ok(); + + /* during test, do not create "unique names" (i.e., names with path + * hashes), so we get a predictable result */ + const auto unique_names{!g_getenv("MU_TEST")&&!g_test_initialized()}; + + if (auto&& res = maildir_link(msg->path(), opts.find.linksdir, unique_names); !res) + return Err(std::move(res.error())); + + return Ok(); +} + +static void +ansi_color_maybe(Field::Id field_id, bool color) +{ + const char* ansi; + + if (!color) + return; /* nothing to do */ + + switch (field_id) { + case Field::Id::From: ansi = MU_COLOR_CYAN; break; + + case Field::Id::To: + case Field::Id::Cc: + case Field::Id::Bcc: ansi = MU_COLOR_BLUE; break; + case Field::Id::Subject: ansi = MU_COLOR_GREEN; break; + case Field::Id::Date: ansi = MU_COLOR_MAGENTA; break; + + default: + if (field_from_id(field_id).type != Field::Type::String) + ansi = MU_COLOR_YELLOW; + else + ansi = MU_COLOR_RED; + } + + fputs(ansi, stdout); +} + +static void +ansi_reset_maybe(Field::Id field_id, bool color) +{ + if (!color) + return; /* nothing to do */ + + fputs(MU_COLOR_DEFAULT, stdout); +} + +static std::string +display_field(const Message& msg, Field::Id field_id) +{ + switch (field_from_id(field_id).type) { + case Field::Type::String: + return msg.document().string_value(field_id); + case Field::Type::Integer: + if (field_id == Field::Id::Priority) { + return to_string(msg.priority()); + } else if (field_id == Field::Id::Flags) { + return to_string(msg.flags()); + } else /* as string */ + return msg.document().string_value(field_id); + case Field::Type::TimeT: + return mu_format("{:%c}", + mu_time(msg.document().integer_value(field_id))); + case Field::Type::ByteSize: + return to_string(msg.document().integer_value(field_id)); + case Field::Type::StringList: + return join(msg.document().string_vec_value(field_id), ','); + case Field::Type::ContactList: + return to_string(msg.document().contacts_value(field_id)); + default: + g_return_val_if_reached(""); + return ""; + } +} + +static void +print_summary(const Message& msg, const Options& opts) +{ + const auto body{msg.body_text()}; + if (!body) + return; + + const auto summ{summarize(body->c_str(), opts.find.summary_len.value_or(0))}; + + mu_print("Summary: "); + fputs_encoded(summ, stdout); + mu_println(""); +} + +static void +thread_indent(const QueryMatch& info, const Options& opts) +{ + const auto is_root{any_of(info.flags & QueryMatch::Flags::Root)}; + const auto first_child{any_of(info.flags & QueryMatch::Flags::First)}; + const auto last_child{any_of(info.flags & QueryMatch::Flags::Last)}; + const auto empty_parent{any_of(info.flags & QueryMatch::Flags::Orphan)}; + const auto is_dup{any_of(info.flags & QueryMatch::Flags::Duplicate)}; + // const auto is_related{any_of(info.flags & QueryMatch::Flags::Related)}; + + /* indent */ + if (opts.debug) { + ::fputs(info.thread_path.c_str(), stdout); + ::fputs(" ", stdout); + } else + for (auto i = info.thread_level; i > 1; --i) + ::fputs(" ", stdout); + + if (!is_root) { + if (first_child) + ::fputs("\\", stdout); + else if (last_child) + ::fputs("/", stdout); + else + ::fputs(" ", stdout); + ::fputs(empty_parent ? "*> " : is_dup ? "=> " + : "-> ", + stdout); + } +} + +static void +output_plain_fields(const Message& msg, const std::string& fields, + bool color, bool threads) +{ + size_t nonempty{}; + + for (auto&& k: fields) { + const auto field_opt{field_from_shortcut(k)}; + if (!field_opt || (!field_opt->is_value() && !field_opt->is_contact())) + nonempty += printf("%c", k); + + else { + ansi_color_maybe(field_opt->id, color); + nonempty += fputs_encoded( + display_field(msg, field_opt->id), stdout); + ansi_reset_maybe(field_opt->id, color); + } + } + + if (nonempty) + fputs("\n", stdout); +} + +static Result<void> +output_plain(const Option<Message>& msg, const OutputInfo& info, + const Options& opts) +{ + if (!msg) + return Ok(); + + /* we reuse the color (whatever that may be) + * for message-priority for threads, too */ + ansi_color_maybe(Field::Id::Priority, !opts.nocolor); + if (opts.find.threads && info.match_info) + thread_indent(*info.match_info, opts); + + output_plain_fields(*msg, opts.find.fields, !opts.nocolor, opts.find.threads); + + if (opts.view.summary_len) + print_summary(*msg, opts); + + return Ok(); +} + +static Result<void> +output_sexp(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (msg) { + if (const auto sexp{msg->sexp()}; !sexp.empty()) + fputs(sexp.to_string().c_str(), stdout); + else + fputs(msg->sexp().to_string().c_str(), stdout); + fputs("\n", stdout); + } + + return Ok(); +} + +static Result<void> +output_json(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (info.header) { + mu_println("["); + return Ok(); + } + + if (info.footer) { + mu_println("]"); + return Ok(); + } + + if (!msg) + return Ok(); + + mu_println("{}{}", msg->sexp().to_json_string(), info.last ? "" : ","); + + return Ok(); +} + +static void +print_attr_xml(const std::string& elm, const std::string& str) +{ + if (str.empty()) + return; /* empty: don't include */ + + auto&& esc{to_string_opt_gchar(g_markup_escape_text(str.c_str(), -1))}; + mu_println("\t\t<{}>{}</{}>", elm, esc.value_or(""), elm); +} + +static Result<void> +output_xml(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (info.header) { + mu_println("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"); + mu_println("<messages>"); + return Ok(); + } + + if (info.footer) { + mu_println("</messages>"); + return Ok(); + } + + mu_println("\t<message>"); + print_attr_xml("from", to_string(msg->from())); + print_attr_xml("to", to_string(msg->to())); + print_attr_xml("cc", to_string(msg->cc())); + print_attr_xml("subject", msg->subject()); + mu_println("\t\t<date>{}</date>", (unsigned)msg->date()); + mu_println("\t\t<size>{}</size>", (unsigned)msg->size()); + print_attr_xml("msgid", msg->message_id()); + print_attr_xml("path", msg->path()); + print_attr_xml("maildir", msg->maildir()); + mu_println("\t</message>"); + + return Ok(); +} + +static OutputFunc +get_output_func(const Options& opts) +{ + if (!opts.find.exec.empty()) + return exec_cmd; + + switch (opts.find.format) { + case Format::Links: + return output_link; + case Format::Plain: + return output_plain; + case Format::Xml: + return output_xml; + case Format::Sexp: + return output_sexp; + case Format::Json: + return output_json; + default: + throw Error(Error::Code::Internal, + "invalid format {}", + static_cast<size_t>(opts.find.format)); + } +} + +static Result<void> +output_query_results(const QueryResults& qres, const Options& opts) +{ + GError* err{}; + const auto output_func{get_output_func(opts)}; + if (!output_func) + return Err(Error::Code::Query, &err, "failed to find output function"); + + if (auto&& res = output_func(Nothing, FirstOutput, opts); !res) + return Err(std::move(res.error())); + + size_t n{0}; + for (auto&& item : qres) { + n++; + auto msg{item.message()}; + if (!msg) + continue; + + if (msg->changed() < opts.find.after.value_or(0)) + continue; + + if (auto&& res = output_func(msg, + {item.doc_id(), + false, + false, + n == qres.size(), /* last? */ + item.query_match()}, + opts); !res) + return Err(std::move(res.error())); + } + + if (auto&& res{output_func(Nothing, LastOutput, opts)}; !res) + return Err(std::move(res.error())); + else + return Ok(); +} + +static Result<void> +process_store_query(const Store& store, const std::string& expr, const Options& opts) +{ + auto qres{run_query(store, expr, opts)}; + if (!qres) + return Err(qres.error()); + + if (qres->empty()) + return Err(Error::Code::NoMatches, "no matches for search expression"); + + return output_query_results(*qres, opts); +} + +Result<void> +Mu::mu_cmd_find(const Store& store, const Options& opts) +{ + auto expr{get_query(store, opts)}; + if (!expr) + return Err(expr.error()); + + if (opts.find.analyze) + return analyze_query_expr(store, *expr, opts); + else + return process_store_query(store, *expr, opts); +} + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + + +/* tests for the command line interface, uses testdir2 */ + +static std::string test_mu_home; + +auto count_nl(const std::string& s)->size_t { + size_t n{}; + for (auto&& c: s) + if (c == '\n') + ++n; + return n; +} + +static size_t +search_func(const std::string& expr, size_t expected) +{ + auto res = run_command({MU_PROGRAM, "find", "--muhome", test_mu_home, expr}); + assert_valid_result(res); + + /* we expect zero lines of error output if there is a match; otherwise + * there should be one line 'No matches found' */ + if (res->exit_code != 0) { + g_assert_cmpuint(res->exit_code, ==, 2); // no match + g_assert_true(res->standard_out.empty()); + g_assert_cmpuint(count_nl(res->standard_err), ==, 1); + return 0; + } + + return count_nl(res->standard_out); +} + +#define search(Q,EXP) do { \ + g_assert_cmpuint(search_func(Q, EXP), ==, EXP); \ +} while(0) + + +static void +test_mu_find_empty_query(void) +{ + search("\"\"", 14); +} + +static void +test_mu_find_01(void) +{ + search("f:john fruit", 1); + search("f:soc@example.com", 1); + search("t:alki@example.com", 1); + search("t:alcibiades", 1); + search("http emacs", 1); + search("f:soc@example.com OR f:john", 2); + search("f:soc@example.com OR f:john OR t:edmond", 3); + search("t:julius", 1); + search("s:dude", 1); + search("t:dantès", 1); +} + +/* index testdir2, and make sure it adds two documents */ +static void +test_mu_find_02(void) +{ + search("bull", 1); + search("g:x", 0); + search("flag:encrypted", 0); + search("flag:attach", 1); + + search("i:3BE9E6535E0D852173@emss35m06.us.lmco.com", 1); +} + +static void +test_mu_find_file(void) +{ + search("file:sittingbull.jpg", 1); + search("file:custer.jpg", 1); + search("file:custer.*", 1); + search("j:sit*", 1); +} + +static void +test_mu_find_mime(void) +{ + search("mime:image/jpeg", 1); + search("mime:text/plain", 14); + search("y:text*", 14); + search("y:image*", 1); + search("mime:message/rfc822", 2); +} + +static void +test_mu_find_text_in_rfc822(void) +{ + search("embed:dancing", 1); + search("e:curious", 1); + search("embed:with", 2); + search("e:karjala", 0); + search("embed:navigation", 1); +} + +static void +test_mu_find_maildir_special(void) +{ + search("\"maildir:/wOm_bàT\"", 3); + search("\"maildir:/wOm*\"", 3); + search("\"maildir:/wOm_*\"", 3); + search("\"maildir:wom_bat\"", 0); + search("\"maildir:/wombat\"", 0); + search("subject:atoms", 1); + search("\"maildir:/wom_bat\" subject:atoms", 1); +} + + +/* some more tests */ + +static void +test_mu_find_wrong_muhome() +{ + auto res = run_command({MU_PROGRAM, "find", "--muhome", + join_paths("/foo", "bar", "nonexistent"), "f:socrates"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==,1); // general error + g_assert_cmpuint(count_nl(res->standard_err), >, 1); +} + +static void +test_mu_find_links(void) +{ + TempDir temp_dir; + + { + auto res = run_command({MU_PROGRAM, "find", "--muhome", test_mu_home, + "--format", "links", "--linksdir", temp_dir.path(), + "mime:message/rfc822"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==,0); + g_assert_cmpuint(count_nl(res->standard_out),==,0); + g_assert_cmpuint(count_nl(res->standard_err),==,0); + } + + + /* furthermore, two symlinks should be there */ + const auto f1{mu_format("{}/cur/rfc822.1", temp_dir)}; + const auto f2{mu_format("{}/cur/rfc822.2", temp_dir)}; + + g_assert_cmpuint(determine_dtype(f1.c_str(), true), ==, DT_LNK); + g_assert_cmpuint(determine_dtype(f2.c_str(), true), ==, DT_LNK); + + /* now we try again, we should get a line of error output, + * when we find the first target file already exists */ + { + auto res = run_command({MU_PROGRAM, "find", "--muhome", test_mu_home, + "--format", "links", "--linksdir", temp_dir.path(), + "mime:message/rfc822"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==,1); + g_assert_cmpuint(count_nl(res->standard_out),==,0); + g_assert_cmpuint(count_nl(res->standard_err),==,1); + } + + /* now we try again with --clearlinks, and the we should be + * back to 0 errors */ + { + auto res = run_command({MU_PROGRAM, "find", "--muhome", test_mu_home, + "--format", "links", "--clearlinks", "--linksdir", temp_dir.path(), + "mime:message/rfc822"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==,0); + g_assert_cmpuint(count_nl(res->standard_out),==,0); + g_assert_cmpuint(count_nl(res->standard_err),==,0); + } + + g_assert_cmpuint(determine_dtype(f1.c_str(), true), ==, DT_LNK); + g_assert_cmpuint(determine_dtype(f2.c_str(), true), ==, DT_LNK); +} + +/* some more tests */ + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + if (!set_en_us_utf8_locale()) + return 0; /* don't error out... */ + + TempDir temp_dir{}; + { + test_mu_home = temp_dir.path(); + + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", + "--muhome", test_mu_home, "--maildir" , MU_TESTMAILDIR2}); + assert_valid_result(res1); + + auto res2 = run_command({MU_PROGRAM, "--quiet", "index", + "--muhome", test_mu_home}); + assert_valid_result(res2); + } + + g_test_add_func("/cmd/find/empty-query", test_mu_find_empty_query); + g_test_add_func("/cmd/find/01", test_mu_find_01); + g_test_add_func("/cmd/find/02", test_mu_find_02); + g_test_add_func("/cmd/find/file", test_mu_find_file); + g_test_add_func("/cmd/find/mime", test_mu_find_mime); + g_test_add_func("/cmd/find/links", test_mu_find_links); + g_test_add_func("/cmd/find/text-in-rfc822", test_mu_find_text_in_rfc822); + g_test_add_func("/cmd/find/wrong-muhome", test_mu_find_wrong_muhome); + g_test_add_func("/cmd/find/maildir-special", test_mu_find_maildir_special); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-index.cc b/mu/mu-cmd-index.cc new file mode 100644 index 0000000..6b3b255 --- /dev/null +++ b/mu/mu-cmd-index.cc @@ -0,0 +1,201 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include <chrono> +#include <thread> +#include <atomic> + +#include <errno.h> +#include <string.h> +#include <cstdio> +#include <signal.h> +#include <unistd.h> + +#include "mu-store.hh" + +using namespace Mu; + +static std::atomic<bool> caught_signal; + +static void +sig_handler(int _sig) +{ + caught_signal = true; +} + +static void +install_sig_handler(void) +{ + struct sigaction action; + int i, sigs[] = {SIGINT, SIGHUP, SIGTERM}; + + sigemptyset(&action.sa_mask); + action.sa_flags = SA_RESETHAND; + action.sa_handler = sig_handler; + + for (i = 0; i != G_N_ELEMENTS(sigs); ++i) + if (sigaction(sigs[i], &action, NULL) != 0) + mu_critical("set sigaction for {} failed: {}", + sigs[i], g_strerror(errno)); +} + +static void +print_stats(const Indexer::Progress& stats, bool color) +{ + const char* kars = "-\\|/"; + static auto i = 0U; + + MaybeAnsi col{color}; + using Color = MaybeAnsi::Color; + + mu_print("{}{}{} indexing messages; " + "checked: {}{}{}; " + "updated/new: {}{}{}; " + "cleaned-up: {}{}{}", + col.fg(Color::Yellow), kars[++i % 4], col.reset(), + col.fg(Color::Green), static_cast<size_t>(stats.checked), col.reset(), + col.fg(Color::Green), static_cast<size_t>(stats.updated), col.reset(), + col.fg(Color::Green), static_cast<size_t>(stats.removed), col.reset()); +} + +Result<void> +Mu::mu_cmd_index(const Options& opts) +{ + auto store = std::invoke([&]{ + if (opts.index.reindex) + return Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::ReInit|Store::Options::Writable); + else + return Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::Writable); + }); + + if (!store) + return Err(store.error()); + + const auto mdir{store->root_maildir()}; + if (G_UNLIKELY(::access(mdir.c_str(), R_OK) != 0)) + return Err(Error::Code::File, "'{}' is not readable: {}", + mdir, g_strerror(errno)); + + MaybeAnsi col{!opts.nocolor}; + using Color = MaybeAnsi::Color; + if (!opts.quiet) { + if (opts.index.lazycheck) + mu_print("lazily "); + + mu_println("indexing maildir {}{}{} -> " + "store {}{}{}", + col.fg(Color::Green), store->root_maildir(), col.reset(), + col.fg(Color::Blue), store->path(), col.reset()); + } + + Mu::Indexer::Config conf{}; + conf.cleanup = !opts.index.nocleanup; + conf.lazy_check = opts.index.lazycheck; + // ignore .noupdate with an empty store. + conf.ignore_noupdate = store->empty(); + + install_sig_handler(); + + auto& indexer{store->indexer()}; + indexer.start(conf); + while (!caught_signal && indexer.is_running()) { + if (!opts.quiet) + print_stats(indexer.progress(), !opts.nocolor); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + if (!opts.quiet) { + mu_print("\r"); + ::fflush({}); + } + } + + indexer.stop(); + + if (!opts.quiet) { + print_stats(indexer.progress(), !opts.nocolor); + mu_print("\n"); + ::fflush({}); + } + + return Ok(); +} + + +#ifdef BUILD_TESTS + +/* + * Tests. + * + */ +#include <config.h> +#include <mu-store.hh> +#include "utils/mu-test-utils.hh" + + +static void +test_mu_index(size_t batch_size=0) +{ + TempDir temp_dir{}; + + const auto mu_home{temp_dir.path()}; + + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", "--batch-size", + mu_format("{}", batch_size == 0 ? 10000 : batch_size), + "--muhome", mu_home, "--maildir" , MU_TESTMAILDIR2}); + assert_valid_command(res1); + + auto res2 = run_command({MU_PROGRAM, "--quiet", "index", + "--muhome", mu_home}); + assert_valid_command(res2); + + auto&& store = unwrap(Store::make(join_paths(temp_dir.path(), "xapian"))); + g_assert_cmpuint(store.size(),==,14); +} + + +static void +test_mu_index_basic() +{ + test_mu_index(); +} + +static void +test_mu_index_batch() +{ + test_mu_index(2); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/index/basic", test_mu_index_basic); + g_test_add_func("/cmd/index/batch", test_mu_index_batch); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-info.cc b/mu/mu-cmd-info.cc new file mode 100644 index 0000000..2e155fc --- /dev/null +++ b/mu/mu-cmd-info.cc @@ -0,0 +1,316 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include "mu-cmd.hh" +#include <message/mu-message.hh> +#include "utils/mu-utils.hh" + +#include <glib.h> +#include <gmime/gmime.h> + +#include <fmt/ostream.h> + +#include <thirdparty/tabulate.hpp> + +using namespace Mu; +using namespace tabulate; + +template <> struct fmt::formatter<Table> : ostream_formatter {}; + +static void +colorify(Table& table, const Options& opts) +{ + if (opts.nocolor || table.size() == 0) + return; + + for (auto&& c = 0U; c != table.row(0).size(); ++c) { + switch (c) { + case 0: + table.column(c).format() + .font_color(Color::green) + .font_style({FontStyle::bold}); + break; + case 1: + table.column(c).format() + .font_color(Color::blue); + break; + case 2: + table.column(c).format() + .font_color(Color::magenta); + break; + + case 3: + table.column(c).format() + .font_color(Color::yellow); + break; + case 4: + table.column(c).format() + .font_color(Color::green); + break; + case 5: + table.column(c).format() + .font_color(Color::blue); + break; + case 6: + table.column(c).format() + .font_color(Color::magenta); + break; + + case 7: + table.column(c).format() + .font_color(Color::yellow); + break; + default: + table.column(c).format() + .font_color(Color::grey); + break; + } + } + + for (auto&& c = 0U; c != table.row(0).size(); ++c) + table[0][c].format() + .font_color(Color::white) + .font_style({FontStyle::bold}); +} + + +static Result<void> +topic_fields(const Options& opts) +{ + using namespace std::string_literals; + + Table fields; + fields.add_row({"field-name", "alias", "short", "search", + "value", "sexp", "example query", "description"}); + + auto searchable=[&](const Field& field)->std::string { + if (field.is_boolean_term()) + return "boolean"; + if (field.is_phrasable_term()) + return "phrase"; + if (field.is_value()) + return "yes"; + if (field.is_contact()) + return "contact"; + if (field.is_range()) + return "range"; + return "no"; + }; + + size_t row{}; + field_for_each([&](auto&& field){ + if (field.is_internal()) + return; // skip. + + fields.add_row({mu_format("{}", field.name), + field.alias.empty() ? "" : mu_format("{}", field.alias), + field.shortcut ? mu_format("{}", field.shortcut) : ""s, + searchable(field), + field.is_value() ? "yes" : "no", + field.include_in_sexp() ? "yes" : "no", + field.example_query, + field.description}); + ++row; + }); + + colorify(fields, opts); + + std::cout << "# Message fields\n" << fields << '\n'; + + return Ok(); +} + +static Result<void> +topic_flags(const Options& opts) +{ + using namespace tabulate; + using namespace std::string_literals; + + Table flags; + flags.add_row({"flag", "shortcut", "category", "description"}); + + flag_infos_for_each([&](const MessageFlagInfo& info) { + + const auto catname = std::invoke( + [](MessageFlagCategory cat)->std::string { + switch(cat){ + case MessageFlagCategory::Mailfile: + return "file"; + case MessageFlagCategory::Maildir: + return "maildir"; + case MessageFlagCategory::Content: + return "content"; + case MessageFlagCategory::Pseudo: + return "pseudo"; + default: + return {}; + } + }, info.category); + + flags.add_row({mu_format("{}", info.name), + mu_format("{}", info.shortcut), + catname, + std::string{info.description}}); + }); + + colorify(flags, opts); + + std::cout << "# Message flags\n" << flags << '\n'; + + return Ok(); +} + +static Result<void> +topic_store(const Mu::Store& store, const Options& opts) +{ + auto tstamp = [](::time_t t)->std::string { + if (t == 0) + return "never"; + else + return mu_format("{:%c}", mu_time(t)); + }; + + Table info; + const auto conf{store.config()}; + info.add_row({"property", "value"}); + info.add_row({"maildir", store.root_maildir()}); + info.add_row({"database-path", store.path()}); + info.add_row({"schema-version", + mu_format("{}", conf.get<Config::Id::SchemaVersion>())}); + info.add_row({"max-message-size", mu_format("{}", conf.get<Config::Id::MaxMessageSize>())}); + info.add_row({"batch-size", mu_format("{}", conf.get<Config::Id::BatchSize>())}); + info.add_row({"created", tstamp(conf.get<Config::Id::Created>())}); + + for (auto&& c : conf.get<Config::Id::PersonalAddresses>()) + info.add_row({"personal-address", c}); + for (auto&& c : conf.get<Config::Id::IgnoredAddresses>()) + info.add_row({"ignored-address", c}); + + info.add_row({"messages in store", mu_format("{}", store.size())}); + info.add_row({"support-ngrams", conf.get<Config::Id::SupportNgrams>() ? "yes" : "no"}); + + info.add_row({"last-change", tstamp(store.statistics().last_change)}); + info.add_row({"last-index", tstamp(store.statistics().last_index)}); + + if (!opts.nocolor) + colorify(info, opts); + + std::cout << info << '\n'; + + return Ok(); +} + +static Result<void> +topic_maildirs(const Mu::Store& store, const Options& opts) +{ + for (auto&& mdir: store.maildirs()) + mu_println("{}", mdir); + + return Ok(); +} + +static Result<void> +topic_mu(const Options& opts) +{ + Table info; + + using namespace tabulate; + + info.add_row({"property", "value", "description"}); + info.add_row({"mu-version", std::string{VERSION}, "Mu runtime version"}); + info.add_row({"xapian-version", Xapian::version_string(), "Xapian runtime version"}); + info.add_row({"gmime-version", + mu_format("{}.{}.{}", gmime_major_version, gmime_minor_version, + gmime_micro_version), "GMime runtime version"}); + info.add_row({"glib-version", + mu_format("{}.{}.{}", glib_major_version, glib_minor_version, + glib_micro_version), "GLib runtime version"}); + info.add_row({"schema-version", mu_format("{}", MU_STORE_SCHEMA_VERSION), + "Version of mu's database schema"}); + + info.add_row({"cld2-support", +#if HAVE_CLD2 + "yes" +#else + "no" +#endif + , "Support searching by language-code?"}); + + info.add_row({"guile-support", +#if BUILD_GUILE + "yes" +#else + "no" +#endif + , "GNU Guile 3.x scripting support?"}); + info.add_row({"readline-support", +#if HAVE_LIBREADLINE + "yes" +#else + "no" +#endif + , "Better 'm server' REPL for debugging?"}); + + if (!opts.nocolor) + colorify(info, opts); + + std::cout << info << '\n'; + + return Ok(); +} + + +Result<void> +Mu::mu_cmd_info(const Mu::Store& store, const Options& opts) +{ + if (!locale_workaround()) + return Err(Error::Code::User, "failed to find a working locale"); + + const auto topic{opts.info.topic}; + if (topic == "store") + return topic_store(store, opts); + else if (topic == "maildirs") + return topic_maildirs(store, opts); + else if (topic == "fields") { + topic_fields(opts); + std::cout << std::endl; + return topic_flags(opts); + } else if (topic == "mu") { + return topic_mu(opts); + } else { + topic_mu(opts); + + MaybeAnsi col{!opts.nocolor}; + using Color = MaybeAnsi::Color; + + auto topic = [&](auto&& t, auto&& d)->std::string { + return mu_format("{}{:<10}{} - {:>12}", + col.fg(Color::Green), t, col.reset(), d); + }; + + mu_println("\nother info topics ('mu info <topic>'):\n{}\n{}\n{}", + topic("store", "information about the message store (database)"), + topic("maildirs", "list the maildirs under the store's root-maildir"), + topic("fields", "information about message fields")); + } + + return Ok(); +} diff --git a/mu/mu-cmd-init.cc b/mu/mu-cmd-init.cc new file mode 100644 index 0000000..26a9600 --- /dev/null +++ b/mu/mu-cmd-init.cc @@ -0,0 +1,144 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include "mu-cmd.hh" + +using namespace Mu; + +#ifndef BUILD_TESTS + +Result<void> +Mu::mu_cmd_init(const Options& opts) +{ + auto store = std::invoke([&]()->Result<Store> { + + /* + * reinit + */ + if (opts.init.reinit) + return Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::ReInit|Store::Options::Writable); + /* + * full init + */ + + /* not provided, nor could we find a good default */ + if (opts.init.maildir.empty()) + return Err(Error::Code::InvalidArgument, + "missing --maildir parameter and could " + "not determine default"); + else if (!g_path_is_absolute(opts.init.maildir.c_str())) + return Err(Error{Error::Code::File, + "--maildir is not absolute"}); + + MemDb mdb; + Config conf{mdb}; + + if (opts.init.max_msg_size) + conf.set<Config::Id::MaxMessageSize>(*opts.init.max_msg_size); + if (opts.init.batch_size && *opts.init.batch_size != 0) + conf.set<Config::Id::BatchSize>(*opts.init.batch_size); + if (!opts.init.my_addresses.empty()) + conf.set<Config::Id::PersonalAddresses>(opts.init.my_addresses); + if (!opts.init.ignored_addresses.empty()) + conf.set<Config::Id::IgnoredAddresses>(opts.init.ignored_addresses); + if (opts.init.support_ngrams) + conf.set<Config::Id::SupportNgrams>(true); + + return Store::make_new(opts.runtime_path(RuntimePath::XapianDb), + opts.init.maildir, conf); + }); + + if (!store) + return Err(store.error()); + + if (!opts.quiet) { + + mu_println("mu has been {} with the following properties:", + opts.init.reinit ? "reinitialized" : "created"); + // mildly hacky + Options opts_copy{opts}; + opts_copy.info.topic = "store"; + mu_cmd_info(*store, opts_copy); + + mu_println("Database is empty. You can use 'mu index' to fill it."); + } + + return Ok(); +} + + + +#else /* BUILD_TESTS */ + +/* + * Tests. + * + */ +#include <config.h> +#include <mu-store.hh> +#include "utils/mu-test-utils.hh" + + +static void +test_mu_init_basic() +{ + TempDir temp_dir{}; + + const auto mu_home{temp_dir.path()}; + + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", + "--muhome", mu_home, "--maildir" , MU_TESTMAILDIR2}); + assert_valid_command(res1); + + auto&& store = unwrap(Store::make(join_paths(temp_dir.path(), "xapian"))); + g_assert_true(store.empty()); +} + +static void +test_mu_init_maildir() +{ + TempDir temp_dir{}; + + const auto mu_home{temp_dir.path()}; + + g_setenv("MAILDIR", MU_TESTMAILDIR2, 1); + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", + "--muhome", mu_home}); + assert_valid_command(res1); + + auto&& store = unwrap(Store::make(join_paths(temp_dir.path(), "xapian"))); + g_assert_true(store.empty()); + assert_equal(store.root_maildir(), MU_TESTMAILDIR2); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/init/basic", test_mu_init_basic); + g_test_add_func("/cmd/init/maildir", test_mu_init_maildir); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-mkdir.cc b/mu/mu-cmd-mkdir.cc new file mode 100644 index 0000000..b91bdec --- /dev/null +++ b/mu/mu-cmd-mkdir.cc @@ -0,0 +1,99 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include "mu-maildir.hh" + +using namespace Mu; + +Mu::Result<void> +Mu::mu_cmd_mkdir(const Options& opts) +{ + for (auto&& dir: opts.mkdir.dirs) { + if (auto&& res = + maildir_mkdir(dir, opts.mkdir.mode); !res) + return res; + } + + return Ok(); +} + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_mkdir_single() +{ + auto testroot{unwrap(make_temp_dir())}; + auto testdir1{join_paths(testroot, "testdir1")}; + + auto res = run_command({MU_PROGRAM, "mkdir", testdir1}); + assert_valid_command(res); + + g_assert_true(check_dir(join_paths(testdir1, "cur"), true, true)); + g_assert_true(check_dir(join_paths(testdir1, "new"), true, true)); + g_assert_true(check_dir(join_paths(testdir1, "tmp"), true, true)); +} + +static void +test_mkdir_multi() +{ + auto testroot{unwrap(make_temp_dir())}; + auto testdir2{join_paths(testroot, "testdir2")}; + auto testdir3{join_paths(testroot, "testdir3")}; + + auto res = run_command({MU_PROGRAM, "mkdir", testdir2, testdir3}); + assert_valid_command(res); + + g_assert_true(check_dir(join_paths(testdir2, "cur"), true, true)); + g_assert_true(check_dir(join_paths(testdir2, "new"), true, true)); + g_assert_true(check_dir(join_paths(testdir3, "tmp"), true, true)); + + g_assert_true(check_dir(join_paths(testdir3, "cur"), true, true)); + g_assert_true(check_dir(join_paths(testdir3, "new"), true, true)); + g_assert_true(check_dir(join_paths(testdir3, "tmp"), true, true)); +} + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/mkdir/single", test_mkdir_single); + g_test_add_func("/cmd/mkdir/multi", test_mkdir_multi); + + return g_test_run(); + +} catch (const Error& e) { + mu_printerrln("{}", e.what()); + return 1; +} catch (...) { + mu_printerrln("caught exception"); + return 1; +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-move.cc b/mu/mu-cmd-move.cc new file mode 100644 index 0000000..7ad7a48 --- /dev/null +++ b/mu/mu-cmd-move.cc @@ -0,0 +1,273 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include "mu-store.hh" +#include "mu-maildir.hh" +#include "message/mu-message-file.hh" + + #include <unistd.h> + +using namespace Mu; + + +Result<void> +Mu::mu_cmd_move(Mu::Store& store, const Options& opts) +{ + const auto& src{opts.move.src}; + if (::access(src.c_str(), R_OK) != 0 || determine_dtype(src) != DT_REG) + return Err(Error::Code::InvalidArgument, + "Source is not a readable file"); + + auto id{store.find_message_id(src)}; + if (!id) + return Err(Error{Error::Code::InvalidArgument, + "Source file is not present in database"} + .add_hint("Perhaps run mu index?")); + + std::string dest{opts.move.dest}; + Option<const std::string&> dest_path; + if (dest.empty() && opts.move.flags.empty()) + return Err(Error::Code::InvalidArgument, + "Must have at least one of destination and flags"); + else if (!dest.empty()) { + const auto mdirs{store.maildirs()}; + + if (!seq_some(mdirs, [&](auto &&d){ return d == dest;})) + return Err(Error{Error::Code::InvalidArgument, + "No maildir '{}' in store", dest} + .add_hint("Try 'mu mkdir'")); + else + dest_path = dest; + } + + auto old_flags{flags_from_path(src)}; + if (!old_flags) + return Err(Error::Code::InvalidArgument, "failed to determine old flags"); + + Flags new_flags{}; + if (!opts.move.flags.empty()) { + if (auto&& nflags{flags_from_expr(to_string_view(opts.move.flags), + *old_flags)}; !nflags) + return Err(Error::Code::InvalidArgument, "Invalid flags"); + else + new_flags = flags_maildir_file(*nflags); + + if (any_of(new_flags & Flags::New) && new_flags != Flags::New) + return Err(Error{Error::Code::File, + "the New flag cannot be combined with others"} + .add_hint("See the mu-move manpage")); + } + + Store::MoveOptions move_opts{}; + if (opts.move.change_name) + move_opts |= Store::MoveOptions::ChangeName; + if (opts.move.update_dups) + move_opts |= Store::MoveOptions::DupFlags; + if (opts.move.dry_run) + move_opts |= Store::MoveOptions::DryRun; + + auto id_paths = store.move_message(*id, dest_path, new_flags, move_opts); + if (!id_paths) + return Err(std::move(id_paths.error())); + + for (const auto&[_id, path]: *id_paths) + mu_println("{}", path); + + return Ok(); +} + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_move_dry_run() +{ + allow_warnings(); + + TempDir tdir; + const auto dbpath{runtime_path(RuntimePath::XapianDb, tdir.path())}; + + auto res = run_command0({CP_PROGRAM, "-r", MU_TESTMAILDIR, tdir.path()}); + assert_valid_command(res); + + const auto testpath{join_paths(tdir.path(), "testdir")}; + const auto src{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,S")}; + { + auto store = Store::make_new(dbpath, testpath, {}); + assert_valid_result(store); + g_assert_true(store->indexer().start({}, true/*block*/)); + } + + // make a message 'New' + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "N", "--dry-run"}); + assert_valid_command(res); + + auto dst{join_paths(testpath, "new", "1220863042.12663_1.mindcrime")}; + assert_equal(res->standard_out, dst + '\n'); + + g_assert_true(::access(dst.c_str(), F_OK) != 0); + g_assert_true(::access(src.c_str(), F_OK) == 0); + } + + // change some flags + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "FP", "--dry-run"}); + assert_valid_command(res); + + auto dst{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,FP")}; + assert_equal(res->standard_out, dst + '\n'); + } + + // change some relative flag + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "+F", "--dry-run"}); + assert_valid_command(res); + + auto dst{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,FS")}; + assert_equal(res->standard_out, dst + '\n'); + } + + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "-S+P+T", "--dry-run"}); + assert_valid_command(res); + + auto dst{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,PT")}; + assert_equal(res->standard_out, dst + '\n'); + } + + // change maildir + for (auto& o : {"o1", "o2"}) + assert_valid_result(maildir_mkdir(join_paths(tdir.path(), "testdir", o))); + + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "/o1", "--flags", "-S+F", "--dry-run"}); + assert_valid_command(res); + assert_equal(res->standard_out, + join_paths(testpath, + "o1/cur", "1220863042.12663_1.mindcrime!2,F") + "\n"); + } + + // change-dups; first create some dups and index them. + assert_valid_result(run_command0({CP_PROGRAM, src, join_paths(testpath, "o1/cur")})); + assert_valid_result(run_command0({CP_PROGRAM, src, join_paths(testpath, "o2/cur")})); + { + auto store = Store::make(dbpath, Store::Options::Writable); + assert_valid_result(store); + g_assert_true(store->indexer().start({}, true/*block*/)); + } + + // change some flags + update dups + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "-S+S+T+R", "--update-dups", "--dry-run"}); + assert_valid_command(res); + + auto p{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,RST")}; + auto p1{join_paths(testpath, "o1", "cur", "1220863042.12663_1.mindcrime!2,RS")}; + auto p2{join_paths(testpath, "o2", "cur", "1220863042.12663_1.mindcrime!2,RS")}; + + assert_equal(res->standard_out, mu_format("{}\n{}\n{}\n", p, p1, p2)); + } +} + + +static void +test_move_real() +{ + allow_warnings(); + + TempDir tdir; + const auto dbpath{runtime_path(RuntimePath::XapianDb, tdir.path())}; + + auto res = run_command0({CP_PROGRAM, "-r", MU_TESTMAILDIR, tdir.path()}); + assert_valid_command(res); + + const auto testpath{join_paths(tdir.path(), "testdir")}; + const auto src{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,S")}; + { + auto store = Store::make_new(dbpath, testpath, {}); + assert_valid_result(res); + g_assert_true(store->indexer().start({}, true/*block*/)); + } + + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "N"}); + assert_valid_command(res); + auto dst{join_paths(testpath, "new", "1220863042.12663_1.mindcrime")}; + g_assert_true(::access(dst.c_str(), F_OK) == 0); + g_assert_true(::access(src.c_str(), F_OK) != 0); + } + + // change flags, maildir, update-dups + // change-dups; first create some dups and index them. + const auto src2{join_paths(testpath, "cur", "1305664394.2171_402.cthulhu!2,")}; + for (auto& o : {"o1", "o2", "o3"}) + assert_valid_result(maildir_mkdir(join_paths(tdir.path(), "testdir", o))); + assert_valid_result(run_command0({CP_PROGRAM, src2, join_paths(testpath, "o1/cur")})); + assert_valid_result(run_command0({CP_PROGRAM, src2, join_paths(testpath, "o2/new")})); + { + auto store = Store::make(dbpath, Store::Options::Writable); + assert_valid_result(store); + g_assert_true(store->indexer().start({}, true/*block*/)); + } + + auto res2 = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src2, "/o3", + "--flags", "-S+S+T+R", "--update-dups", "--change-name"}); + assert_valid_command(res2); + + auto store = Store::make(dbpath, Store::Options::Writable); + assert_valid_result(store); + g_assert_true(store->indexer().start({}, true/*block*/)); + + for (auto&& f: split(res2->standard_out, "\n")) { + //mu_println(">> {}", f); + if (f.length() > 2) + g_assert_true(::access(f.c_str(), F_OK) == 0); + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/move/dry-run", test_move_dry_run); + g_test_add_func("/cmd/move/real", test_move_real); + + return g_test_run(); + +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-remove.cc b/mu/mu-cmd-remove.cc new file mode 100644 index 0000000..5eb96b8 --- /dev/null +++ b/mu/mu-cmd-remove.cc @@ -0,0 +1,100 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +using namespace Mu; + +Result<void> +Mu::mu_cmd_remove(Mu::Store& store, const Options& opts) +{ + for (auto&& file: opts.remove.files) { + const auto res = store.remove_message(file); + if (!res) + return Err(Error::Code::File, "failed to remove {}", file.c_str()); + else + mu_debug("removed message @ {}", file); + } + + return Ok(); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_remove_ok() +{ + auto testhome{unwrap(make_temp_dir())}; + auto dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + + /* create a writable copy */ + const auto testmdir = join_paths(testhome, "test-maildir"); + const auto testmsg = join_paths(testmdir, "/cur/1220863042.12663_1.mindcrime!2,S"); + auto cres = run_command({CP_PROGRAM, "-r", MU_TESTMAILDIR, testmdir}); + assert_valid_command(cres); + + { + auto&& store = unwrap(Store::make_new(dbpath, testmdir)); + auto res = store.add_message(testmsg); + assert_valid_result(res); + g_assert_true(store.contains_message(testmsg)); + } + + { // remove the same + auto res = run_command({MU_PROGRAM, "remove", + mu_format("--muhome={}", testhome), + testmsg}); + assert_valid_command(res); + } + + { + auto&& store = unwrap(Store::make(dbpath)); + g_assert_false(!!store.contains_message(testmsg)); + g_assert_cmpuint(::access(testmsg.c_str(), F_OK), ==, 0); + } + + remove_directory(testhome); +} + + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/remove/ok", test_remove_ok); + + return g_test_run(); + +} catch (const Error& e) { + mu_printerrln("{}", e.what()); + return 1; +} catch (...) { + mu_printerrln("caught exception"); + return 1; +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-script.cc b/mu/mu-cmd-script.cc new file mode 100644 index 0000000..2302dd7 --- /dev/null +++ b/mu/mu-cmd-script.cc @@ -0,0 +1,49 @@ +/* +** Copyright (C) 2012-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include "mu-cmd.hh" +#include "mu-script.hh" +#include "utils/mu-utils.hh" + +using namespace Mu; + +Result<void> +Mu::mu_cmd_script(const Options& opts) +{ + ScriptPaths paths = { MU_SCRIPTS_DIR }; + const auto&& scriptinfos{script_infos(paths)}; + auto script_it = Mu::seq_find_if(scriptinfos, [&](auto&& item) { + return item.name == opts.script.name; + }); + + if (script_it == scriptinfos.cend()) + return Err(Error::Code::InvalidArgument, + "cannot find script '{}'", opts.script.name); + + std::vector<std::string> params{opts.script.params}; + if (!opts.muhome.empty()) { + params.emplace_back("--muhome"); + params.emplace_back(opts.muhome); + } + + // won't return unless there's an error. + return run_script(script_it->path, opts.script.params); +} diff --git a/mu/mu-cmd-server.cc b/mu/mu-cmd-server.cc new file mode 100644 index 0000000..3a69456 --- /dev/null +++ b/mu/mu-cmd-server.cc @@ -0,0 +1,164 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <string> +#include <algorithm> +#include <atomic> +#include <cstdio> + +#include <unistd.h> + +#include "mu-cmd.hh" +#include "mu-server.hh" + +#include "utils/mu-utils.hh" +#include "utils/mu-readline.hh" + +using namespace Mu; + +static std::atomic<int> MuTerminate{0}; +static bool tty; + +static void +sig_handler(int sig) +{ + MuTerminate = sig; +} + +static void +install_sig_handler() +{ + MuTerminate = 0; + + struct sigaction action{}; + action.sa_handler = sig_handler; + sigemptyset(&action.sa_mask); + action.sa_flags = SA_RESETHAND; + + for (auto sig: {SIGINT, SIGHUP, SIGTERM, SIGPIPE}) + if (sigaction(sig, &action, NULL) != 0) + mu_critical("set sigaction for {} failed: {}", + sig, g_strerror(errno)); +} + +/* + * Markers for/after the length cookie that precedes the expression we write to + * output. We use octal 376, 377 (ie, 0xfe, 0xff) as they will never occur in + * utf8 + */ + +#define COOKIE_PRE "\376" +#define COOKIE_POST "\377" + +static void +cookie(size_t n) +{ + const auto num{static_cast<unsigned>(n)}; + + if (tty) // for testing. + ::printf("[%x]", num); + else + ::printf(COOKIE_PRE "%x" COOKIE_POST, num); +} + + + +static void +output_stdout(const std::string& str, Server::OutputFlags flags) +{ + cookie(str.size() + 1); + if (G_UNLIKELY(::puts(str.c_str()) < 0)) { + mu_critical("failed to write output '{}'", str); + ::raise(SIGTERM); /* terminate ourselves */ + } + if (any_of(flags & Server::OutputFlags::Flush)) + std::fflush(stdout); +} + + +static void +report_error(const Mu::Error& err) noexcept +{ + output_stdout(Sexp(":error"_sym, Error::error_number(err.code()), + ":message"_sym, err.what()).to_string(), + Server::OutputFlags::Flush); +} + +Result<void> +Mu::mu_cmd_server(const Mu::Options& opts) try { + + auto store = Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::Writable); + if (!store) + return Err(store.error()); + + Server::Options sopts{}; + sopts.allow_temp_file = opts.server.allow_temp_file; + + Server server{*store, sopts, output_stdout}; + mu_message("created server with store @ {}; maildir @ {}; debug-mode {};" + "readline: {}", + store->path(), store->root_maildir(), + opts.debug ? "yes" : "no", + have_readline() ? "yes" : "no"); + + tty = ::isatty(::fileno(stdout)); + const auto eval = std::string{opts.server.commands ? "(help :full t)" : opts.server.eval}; + if (!eval.empty()) { + server.invoke(eval); + return Ok(); + } + + // Note, the readline stuff is inactive unless on a tty. + const auto histpath{opts.runtime_path(RuntimePath::Cache) + "/history"}; + setup_readline(histpath, 50); + + install_sig_handler(); + mu_println(";; Welcome to the " PACKAGE_STRING " command-server{}\n" + ";; Use (help) to get a list of commands, (quit) to quit.", + opts.debug ? " (debug-mode)" : ""); + + bool do_quit{}; + while (!MuTerminate && !do_quit) { + std::fflush(stdout); // Needed for Windows, see issue #1827. + const auto line{read_line(do_quit)}; + if (line.find_first_not_of(" \t") == std::string::npos) + continue; // skip whitespace-only lines + + do_quit = server.invoke(line) ? false : true; + save_line(line); + } + + if (MuTerminate != 0) + mu_message ("shutting down due to signal {}", MuTerminate.load()); + + shutdown_readline(); + + return Ok(); + +} catch (const Error& er) { /* note: user-level error, "OK" for mu */ + report_error(er); + mu_warning("server caught exception: {}", er.what()); + return Ok(); +} catch (...) { + mu_critical("server caught exception"); + return Err(Error::Code::Internal, "caught exception"); +} diff --git a/mu/mu-cmd-verify.cc b/mu/mu-cmd-verify.cc new file mode 100644 index 0000000..7fbb4b9 --- /dev/null +++ b/mu/mu-cmd-verify.cc @@ -0,0 +1,255 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include "message/mu-message.hh" +#include "message/mu-mime-object.hh" + +#include <iostream> +#include <iomanip> + +using namespace Mu; + +template <typename T> +static void +key_val(const Mu::MaybeAnsi& col, const std::string& key, T val) +{ + using Color = Mu::MaybeAnsi::Color; + + mu_println("{}{:<18}{}: {}{}{}", + col.fg(Color::BrightBlue), key, col.reset(), + col.fg(Color::Green), val, col.reset()); +} + +static void +print_signature(const Mu::MimeSignature& sig, const Options& opts) +{ + Mu::MaybeAnsi col{!opts.nocolor}; + + const auto created{sig.created()}; + key_val(col, "created", + created == 0 ? std::string{"unknown"} : + mu_format("{:%c}", mu_time(sig.created()))); + + const auto expires{sig.expires()}; + key_val(col, "expires", expires==0 ? std::string{"never"} : + mu_format("{:%c}", mu_time(sig.expires()))); + + const auto cert{sig.certificate()}; + key_val(col, "public-key algo", + to_string_view_opt(cert.pubkey_algo()).value_or("unknown")); + key_val(col, "digest algo", + to_string_view_opt(cert.digest_algo()).value_or("unknown")); + key_val(col, "id-validity", + to_string_view_opt(cert.id_validity()).value_or("unknown")); + key_val(col, "trust", + to_string_view_opt(cert.trust()).value_or("unknown")); + key_val(col, "issuer-serial", cert.issuer_serial().value_or("unknown")); + key_val(col, "issuer-name", cert.issuer_name().value_or("unknown")); + key_val(col, "finger-print", cert.fingerprint().value_or("unknown")); + key_val(col, "key-id", cert.key_id().value_or("unknown")); + key_val(col, "name", cert.name().value_or("unknown")); + key_val(col, "user-id", cert.user_id().value_or("unknown")); +} + + +static bool +verify(const MimeMultipartSigned& sigpart, const Options& opts) +{ + using VFlags = MimeMultipartSigned::VerifyFlags; + const auto vflags{opts.verify.auto_retrieve ? + VFlags::EnableKeyserverLookups: VFlags::None}; + + auto ctx{MimeCryptoContext::make_gpg()}; + if (!ctx) + return false; + + const auto sigs{sigpart.verify(*ctx, vflags)}; + Mu::MaybeAnsi col{!opts.nocolor}; + + if (!sigs || sigs->empty()) { + + if (!opts.quiet) + mu_println("cannot find signatures in part"); + + return true; + } + + bool valid{true}; + for (auto&& sig: *sigs) { + + const auto status{sig.status()}; + + if (!opts.quiet) + key_val(col, "status", to_string(status)); + + if (opts.verbose) + print_signature(sig, opts); + + if (none_of(sig.status() & MimeSignature::Status::Green)) + valid = false; + } + + return valid; +} + + +static bool +verify_message(const Message& message, const Options& opts, const std::string& name) +{ + if (none_of(message.flags() & Flags::Signed)) { + if (!opts.quiet) + mu_println("{}: no signed parts found", name); + return false; + } + + bool verified{true}; /* innocent until proven guilty */ + for(auto&& part: message.parts()) { + + if (!part.is_signed()) + continue; + + const auto& mobj{part.mime_object()}; + if (!mobj.is_multipart_signed()) + continue; + + if (!verify(MimeMultipartSigned(mobj), opts)) + verified = false; + } + + return verified; +} + + + +Mu::Result<void> +Mu::mu_cmd_verify(const Options& opts) +{ + bool all_ok{true}; + const auto mopts = message_options(opts.verify); + + for (auto&& file: opts.verify.files) { + + auto message{Message::make_from_path(file, mopts)}; + if (!message) + return Err(message.error()); + + if (!opts.quiet && opts.verify.files.size() > 1) + mu_println("verifying {}", file); + + if (!verify_message(*message, opts, file)) + all_ok = false; + } + + // when no messages provided, read from stdin + if (opts.verify.files.empty()) { + const auto msgtxt = read_from_stdin(); + if (!msgtxt) + return Err(msgtxt.error()); + auto message{Message::make_from_text(*msgtxt, {}, mopts)}; + if (!message) + return Err(message.error()); + + all_ok = verify_message(*message, opts, "<stdin>"); + } + + if (all_ok) + return Ok(); + else + return Err(Error::Code::UnverifiedSignature, + "failed to verify one or more signatures"); +} + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +/* we can only test 'verify' if gpg is installed, and has djcb@djcbsoftware's key in the keyring */ +static bool +verify_is_testable(void) +{ + auto gpg{program_in_path("gpg2")}; + if (!gpg) { + mu_message("cannot find gpg2 in path"); + return false; + } + + auto res{run_command({*gpg, "--list-keys", "DCC4A036"})}; /* djcb@djcbsoftware.nl's key */ + if (!res || res->exit_code != 0) { + mu_message("key DCC4A036 not found"); + return false; + } + + return true; +} + +static void +test_mu_verify_good(void) +{ + if (!verify_is_testable()) { + g_test_skip("cannot test verify"); + return; + } + + auto res = run_command({MU_PROGRAM, "verify", + join_paths(MU_TESTMAILDIR4, "signed!2,S")}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code ,==, 0); +} + +static void +test_mu_verify_bad(void) +{ + if (!verify_is_testable()) { + g_test_skip("cannot test verify"); + return; + } + + auto res = run_command({MU_PROGRAM, "verify", + join_paths(MU_TESTMAILDIR4, "signed-bad!2,S")}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==, 1); +} + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/verify/good", test_mu_verify_good); + g_test_add_func("/cmd/verify/bad", test_mu_verify_bad); + + return g_test_run(); + +} catch (const Error& e) { + mu_printerrln("{}", e.what()); + return 1; +} catch (...) { + mu_printerrln("caught exception"); + return 1; +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-view.cc b/mu/mu-cmd-view.cc new file mode 100644 index 0000000..3ddd78e --- /dev/null +++ b/mu/mu-cmd-view.cc @@ -0,0 +1,424 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <config.h> + +#include "mu-cmd.hh" + +#include "message/mu-message.hh" + +#include <iostream> +#include <iomanip> + +using namespace Mu; + + +#define VIEW_TERMINATOR '\f' /* form-feed */ + +using namespace Mu; + +static Mu::Result<void> +view_msg_sexp(const Message& message, const Options& opts) +{ + ::fputs(message.sexp().to_string().c_str(), stdout); + ::fputs("\n", stdout); + + return Ok(); +} + + +static std::string /* return comma-sep'd list of attachments */ +get_attach_str(const Message& message, const Options& opts) +{ + std::string str; + seq_for_each(message.parts(), [&](auto&& part) { + if (auto fname = part.raw_filename(); fname) { + if (str.empty()) + str = fname.value(); + else + str += ", " + fname.value(); + } + }); + + return str; +} + +#define color_maybe(C) \ + do { \ + if (color) \ + fputs((C), stdout); \ + } while (0) + +static void +print_field(const std::string& field, const std::string& val, bool color) +{ + if (val.empty()) + return; + + color_maybe(MU_COLOR_MAGENTA); + fputs_encoded(field, stdout); + color_maybe(MU_COLOR_DEFAULT); + fputs(": ", stdout); + + color_maybe(MU_COLOR_GREEN); + fputs_encoded(val, stdout); + + color_maybe(MU_COLOR_DEFAULT); + fputs("\n", stdout); +} + +/* a summary_len of 0 mean 'don't show summary, show body */ +static void +body_or_summary(const Message& message, const Options& opts) +{ + const auto color{!opts.nocolor}; + using Format = Options::View::Format; + + std::string body, btype; + switch (opts.view.format) { + case Format::Plain: + btype = "plain text"; + body = message.body_text().value_or(""); + break; + case Format::Html: + btype = "html"; + body = message.body_html().value_or(""); + break; + default: + throw std::range_error("unsupported format"); // bug + } + + if (body.empty()) { + if (any_of(message.flags() & Flags::Encrypted)) { + color_maybe(MU_COLOR_CYAN); + mu_println("[No {} body found; message does have encrypted parts]", + btype); + } else { + color_maybe(MU_COLOR_MAGENTA); + mu_println("[No {} body found]", btype); + } + color_maybe(MU_COLOR_DEFAULT); + return; + } + + if (opts.view.summary_len) { + const auto summ{summarize(body, *opts.view.summary_len)}; + print_field("Summary", summ, color); + } else { + mu_print_encoded("{}", body); + if (!g_str_has_suffix(body.c_str(), "\n")) + mu_println(""); + } +} + +/* we ignore fields for now */ +/* summary_len == 0 means "no summary */ +static Mu::Result<void> +view_msg_plain(const Message& message, const Options& opts) +{ + const auto color{!opts.nocolor}; + + print_field("From", to_string(message.from()), color); + print_field("To", to_string(message.to()), color); + print_field("Cc", to_string(message.cc()), color); + print_field("Bcc", to_string(message.bcc()), color); + print_field("Subject", message.subject(), color); + + if (auto&& date = message.date(); date != 0) + print_field("Date", mu_format("{:%c}", mu_time(date)), color); + + print_field("Tags", join(message.tags(), ", "), color); + + print_field("Attachments",get_attach_str(message, opts), color); + + mu_println(""); + body_or_summary(message, opts); + + return Ok(); +} + +static Mu::Result<void> +handle_msg(const Message& message, const Options& opts) +{ + using Format = Options::View::Format; + + switch (opts.view.format) { + case Format::Plain: + case Format::Html: + return view_msg_plain(message, opts); + + case Format::Sexp: + return view_msg_sexp(message, opts); + default: + mu_critical("bug: should not be reached"); + return Err(Error::Code::Internal, "error"); + } +} + +Mu::Result<void> +Mu::mu_cmd_view(const Options& opts) +{ + for (auto&& file: opts.view.files) { + auto message{Message::make_from_path( + file, message_options(opts.view))}; + if (!message) + return Err(message.error()); + + if (auto res = handle_msg(*message, opts); !res) + return res; + /* add a separator between two messages? */ + if (opts.view.terminate) + mu_print("{}", VIEW_TERMINATOR); + } + + // no files? read from stding + if (opts.view.files.empty()) { + const auto msgtxt = read_from_stdin(); + if (!msgtxt) + return Err(msgtxt.error()); + auto message = Message::make_from_text(*msgtxt,{}, + message_options(opts.view)); + if (!message) + return Err(message.error()); + else + return handle_msg(*message, opts); + } + return Ok(); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include <fcntl.h> /* Definition of AT_* constants */ +#include <sys/stat.h> +#include <fstream> +#include <utils/mu-regex.hh> +#include "utils/mu-test-utils.hh" + +static constexpr std::string_view test_msg = +R"(From: Test <test@example.com> +To: abc@example.com +Date: Mon, 23 May 2011 10:53:45 +0200 +Subject: vla +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d" +Message-ID: <10374608.109906.11909.20115aabbccdd.MSGID@mailinglijst.nl> + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/plain; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +text + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/html; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +html + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d-- +)"; + + +static std::string msgpath; + +static void +test_view_plain() +{ + auto res = run_command({MU_PROGRAM, "view", msgpath}); + assert_valid_command(res); + auto output{*res}; + + // silly hack to avoid locale diffs + auto rx = unwrap(Regex::make("^Date:.*", G_REGEX_MULTILINE)); + output.standard_out = unwrap(rx.replace(output.standard_out, "Date: xxx")); + + g_assert_true(output.standard_err.empty()); + assert_equal(output.standard_out, +R"(From: Test <test@example.com> +To: abc@example.com +Subject: vla +Date: xxx + +text +)"); + +} + +static void +test_view_html() +{ + auto res = run_command({MU_PROGRAM, "view", "--format=html", msgpath}); + assert_valid_command(res); + auto output{*res}; + + auto rx = unwrap(Regex::make("^Date:.*", G_REGEX_MULTILINE)); + output.standard_out = unwrap(rx.replace(output.standard_out, "Date: xxx")); + + g_assert_true(output.standard_err.empty()); + assert_equal(output.standard_out, +R"(From: Test <test@example.com> +To: abc@example.com +Subject: vla +Date: xxx + +html +)"); + +} + +static void +test_view_sexp() +{ + TempTz tz("Europe/Amsterdam"); + if (!tz.available()) { + g_test_skip("timezone not available"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", "--format=sexp", msgpath}); + assert_valid_command(res); + auto output{*res}; + + g_assert_true(output.standard_err.empty()); + + // Note: :path, :changed (file ctime) change per run. + struct stat statbuf{}; + g_assert_true(::stat(msgpath.c_str(), &statbuf) == 0); + + const auto expected = mu_format( + R"((:path "{}" :size 638 :changed ({} {} 0) :date (19930 8345 0) :flags (unread) :from ((:email "test@example.com" :name "Test")) :message-id "10374608.109906.11909.20115aabbccdd.MSGID@mailinglijst.nl" :priority normal :subject "vla" :to ((:email "abc@example.com"))) +)", + msgpath, + statbuf.st_ctime >> 16, + statbuf.st_ctime & 0xffff); + + assert_equal(output.standard_out, expected); +} + +static void +test_mu_view_01(void) +{ + TempDir temp_dir{}; + + if (!set_en_us_utf8_locale()) { + g_test_skip("failed to switch to en_US/utf8"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail4")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(res->standard_out.size(), ==, 364); +} + +static void +test_mu_view_multi(void) +{ + TempDir temp_dir{}; + + if (!set_en_us_utf8_locale()) { + g_test_skip("failed to switch to en_US/utf8"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail5"), + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(res->standard_out.size(), ==, 162); +} + +static void +test_mu_view_multi_separate(void) +{ + TempDir temp_dir{}; + + if (!set_en_us_utf8_locale()) { + g_test_skip("failed to switch to en_US/utf8"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", "--terminate", + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail5"), + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(res->standard_out.size(), ==, 164); +} + +static void +test_mu_view_attach(void) +{ + TempDir temp_dir{}; + + if (!set_en_us_utf8_locale()) { + g_test_skip("failed to switch to en_US/utf8"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", "--terminate", + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(res->standard_out.size(), ==, 164); +} + + + +int +main(int argc, char* argv[]) try { + + TempDir tmpdir{}; + msgpath = join_paths(tmpdir.path(), "test-message.txt"); + std::ofstream strm{msgpath}; + strm.write(test_msg.data(), test_msg.size()); + strm.close(); + g_assert_true(strm.good()); + + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/view/01", test_mu_view_01); + g_test_add_func("/cmd/view/multi", test_mu_view_multi); + g_test_add_func("/cmd/view/multi-separate", test_mu_view_multi_separate); + g_test_add_func("/cmd/view/attach", test_mu_view_attach); + g_test_add_func("/cmd/view/plain", test_view_plain); + g_test_add_func("/cmd/view/html", test_view_html); + g_test_add_func("/cmd/view/sexp", test_view_sexp); + + return g_test_run(); + +} catch (const Error& e) { + mu_printerrln("{}", e.what()); + return 1; +} catch (...) { + mu_printerrln("caught exception"); + return 1; +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd.cc b/mu/mu-cmd.cc new file mode 100644 index 0000000..bcdca98 --- /dev/null +++ b/mu/mu-cmd.cc @@ -0,0 +1,169 @@ +/* +** Copyright (C) 2010-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <iostream> +#include <iomanip> + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> + +#include "mu-options.hh" +#include "mu-cmd.hh" +#include "mu-maildir.hh" +#include "mu-contacts-cache.hh" +#include "message/mu-message.hh" +#include "message/mu-mime-object.hh" + +#include "utils/mu-error.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-utils.hh" + +#include <thirdparty/tabulate.hpp> + +using namespace Mu; + + +static Result<void> +cmd_fields(const Options& opts) +{ + mu_printerrln("the 'mu fields' command has been superseded by 'mu info'; try:\n" + " mu info fields\n"); + return Ok(); +} + + +static Result<void> +cmd_find(const Options& opts) +{ + auto store{Store::make(opts.runtime_path(RuntimePath::XapianDb))}; + if (!store) + return Err(store.error()); + else + return mu_cmd_find(*store, opts); +} + + +static void +show_usage(void) +{ + mu_println("usage: mu command [options] [parameters]"); + mu_println("where command is one of index, find, cfind, view, mkdir, " + "extract, add, remove, script, verify or server"); + mu_println("see the mu, mu-<command> or mu-easy manpages for " + "more information"); +} + + +using ReadOnlyStoreFunc = std::function<Result<void>(const Store&, const Options&)>; +using WritableStoreFunc = std::function<Result<void>(Store&, const Options&)>; + +static Result<void> +with_readonly_store(const ReadOnlyStoreFunc& func, const Options& opts) +{ + auto store{Store::make(opts.runtime_path(RuntimePath::XapianDb))}; + if (!store) + return Err(store.error()); + + return func(store.value(), opts); +} + +static Result<void> +with_writable_store(const WritableStoreFunc func, const Options& opts) +{ + auto store{Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::Writable)}; + if (!store) + return Err(store.error()); + + return func(store.value(), opts); +} + +Result<void> +Mu::mu_cmd_execute(const Options& opts) try { + + if (!opts.sub_command) + return Err(Error::Code::Internal, "missing subcommand"); + + switch (*opts.sub_command) { + case Options::SubCommand::Help: + return Ok(); /* already handled in mu-options.cc */ + /* + * no store needed + */ + case Options::SubCommand::Fields: + return cmd_fields(opts); + case Options::SubCommand::Mkdir: + return mu_cmd_mkdir(opts); + case Options::SubCommand::Script: + return mu_cmd_script(opts); + case Options::SubCommand::View: + return mu_cmd_view(opts); + case Options::SubCommand::Verify: + return mu_cmd_verify(opts); + case Options::SubCommand::Extract: + return mu_cmd_extract(opts); + /* + * read-only store + */ + + case Options::SubCommand::Cfind: + return with_readonly_store(mu_cmd_cfind, opts); + case Options::SubCommand::Find: + return cmd_find(opts); + case Options::SubCommand::Info: + return with_readonly_store(mu_cmd_info, opts); + + /* writable store */ + + case Options::SubCommand::Add: + return with_writable_store(mu_cmd_add, opts); + case Options::SubCommand::Remove: + return with_writable_store(mu_cmd_remove, opts); + case Options::SubCommand::Move: + return with_writable_store(mu_cmd_move, opts); + + /* + * commands instantiate store themselves + */ + case Options::SubCommand::Index: + return mu_cmd_index(opts); + case Options::SubCommand::Init: + return mu_cmd_init(opts); + case Options::SubCommand::Server: + return mu_cmd_server(opts); + + default: + show_usage(); + return Ok(); + } + +} catch (const Mu::Error& er) { + return Err(er); +} catch (const std::runtime_error& re) { + return Err(Error::Code::Internal, "runtime-error: {}", re.what()); +} catch (const std::exception& ex) { + return Err(Error::Code::Internal, "error: {}", ex.what()); +} catch (...) { + return Err(Error::Code::Internal, "caught exception"); +} diff --git a/mu/mu-cmd.hh b/mu/mu-cmd.hh new file mode 100644 index 0000000..7b591f8 --- /dev/null +++ b/mu/mu-cmd.hh @@ -0,0 +1,198 @@ +/* +** Copyright (C) 2008-2022-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_CMD_HH__ +#define MU_CMD_HH__ + +#include <glib.h> +#include <mu-store.hh> +#include <utils/mu-result.hh> + +#include "mu-options.hh" + +namespace Mu { + + +/** + * Get message options from (sub)command options + * + * @param cmdopts (sub) command options + * + * @return message options + */ +template<typename CmdOpts> +constexpr Message::Options +message_options(const CmdOpts& cmdopts) +{ + Message::Options mopts{Message::Options::AllowRelativePath}; + + if (cmdopts.decrypt) + mopts |= Message::Options::Decrypt; + if (cmdopts.auto_retrieve) + mopts |= Message::Options::RetrieveKeys; + + return mopts; +} + +/** + * execute the 'add' command + * + * @param store store object to use + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_add(Store& store, const Options& opts); + +/** + * execute the 'cfind' command + * + * @param store store object to use + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_cfind(const Store& store, const Options& opts); + +/** + * execute the 'extract' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_extract(const Options& opts); + +/** + * execute the 'find' command + * + * @param store store object to use + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_find(const Store& store, const Options& opts); + +/** + * execute the 'index' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_index(const Options& opt); + +/** + * execute the 'info' command + * + * @param store message store object. + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_info(const Mu::Store& store, const Options& opts); + +/** + * execute the 'init' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_init(const Options& opts); + +/** + * execute the 'mkdir' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_mkdir(const Options& opts); + +/** + * execute the 'move' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_move(Store& store, const Options& opts); + +/** + * execute the 'remove' command + * + * @param store store object to use + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_remove(Store& store, const Options& opt); + +/** + * execute the 'script' command + * + * @param opts configuration options + * @param err receives error information, or NULL + * + * @return Ok() or some error + */ +Result<void> mu_cmd_script(const Options& opts); + + +/** + * execute the server command + * @param opts configuration options + * @param err receives error information, or NULL + * + * @return Ok() or some error + */ +Result<void> mu_cmd_server(const Options& opts); + +/** + * execute the 'verify' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Mu::Result<void> mu_cmd_verify(const Options& opts); + +/** + * execute the 'view' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Mu::Result<void> mu_cmd_view(const Options& opts); + +/** + * execute some mu command, based on 'opts' + * + * @param opts configuration option + * @param err receives error information, or NULL + * + * @return Ok() or some error + */ +Result<void> mu_cmd_execute(const Options& opts); + +} // namespace Mu + +#endif /*__MU_CMD_H__*/ diff --git a/mu/mu-memcheck.in b/mu/mu-memcheck.in new file mode 100644 index 0000000..73a2329 --- /dev/null +++ b/mu/mu-memcheck.in @@ -0,0 +1,6 @@ +#!/bin/sh + +export G_SLICE=always-malloc +export G_DEBUG=gc-friendly + +libtool --mode=execute valgrind --tool=memcheck --leak-check=full --show-possibly-lost=no --leak-resolution=med --track-origins=yes --num-callers=20 --log-file='@abs_top_builddir@/mu-%p.vgdump' @abs_top_builddir@/mu/mu $@ diff --git a/mu/mu-options.cc b/mu/mu-options.cc new file mode 100644 index 0000000..9533e9c --- /dev/null +++ b/mu/mu-options.cc @@ -0,0 +1,956 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +/** + * @brief Command-line handling + * + * Here we implement mu's command-line parsing based on the CLI11 library. At + * the time of writing, that library seems to be the best based on the criteria + * that it supports the features we need and is available as a header-only + * include. + * + * CLI11 can do quite a bit, and we're only scratching the surface here, + * plan is to slowly improve things. + * + * - we do quite a bit of sanity-checking, but the errors are a rather terse + * - the docs could be improved, e.g., `mu find --help` and --format/--sortfield + * + */ + + +#include <config.h> +#include <stdexcept> +#include <array> +#include <unordered_map> +#include <iostream> +#include <string_view> +#include <unistd.h> + +#include <utils/mu-utils.hh> +#include <utils/mu-utils-file.hh> +#include <utils/mu-error.hh> +#include "utils/mu-test-utils.hh" +#include "mu-options.hh" +#include "mu-script.hh" + +#include <thirdparty/CLI11.hpp> + +using namespace Mu; + + +/* + * helpers + */ + + + +/** + * array of associated pair elements -- like an alist + * but based on std::array and thus can be constexpr + */ +template<typename T1, typename T2, std::size_t N> + using AssocPairs = std::array<std::pair<T1, T2>, N>; + + +/** + * Get the first value of the pair where the second element is @param s. + * + * @param p AssocPairs + * @param s some second pair value + * + * @return the matching first pair value, or Nothing if not found. + */ +template<typename P> +constexpr Option<typename P::value_type::first_type> +to_first(const P& p, typename P::value_type::second_type s) +{ + for (const auto& item: p) + if (item.second == s) + return item.first; + return Nothing; +} + +/** + * Get the second value of the pair where the first element is @param f. + * + * @param p AssocPairs + * @param f some first pair value + * + * @return the matching second pair value, or Nothing if not found. + */ +template<typename P> +constexpr Option<typename P::value_type::second_type> +to_second(const P& p, typename P::value_type::first_type f) +{ + for (const auto& item: p) + if (item.first == f) + return item.second; + return Nothing; +} + + +/** + * Options-specific array-bases type that maps some enum to a <name, description> pair + */ +template<typename T, std::size_t N> +using InfoEnum = AssocPairs<T, std::pair<std::string_view, std::string_view>, N>; + +/** + * Get the name (shortname) for some InfoEnum, based on the enum + * + * @param ie an InfoEnum + * @param e an enum value + * + * @return the name if found, or Nothing + */ +template<typename IE> +static constexpr Option<std::string_view> +to_name(const IE& ie, typename IE::value_type::first_type e) { + if (auto&& s{to_second(ie, e)}; s) + return s->first; + else + return Nothing; +} + +/** + * Get the enum value for some InfoEnum, based on the name + * + * @param ie an InfoEnum + * @param name some name (shortname) + * + * @return the name if found, or Nothing + */ +template<typename IE> +static constexpr Option<typename IE::value_type::first_type> +to_enum(const IE& ie, std::string_view name) { + for(auto&& item: ie) + if (item.second.first == name) + return item.first; + else + return Nothing; +} + +/** + * List help options for as a string, with the default marked with '(*)' + * + * @param ie infoenum + * @param default_opt default option + * + * @return a help string + */ +template<typename IE> +static std::string +options_help(const IE& ie, typename IE::value_type::first_type default_opt) +{ + std::string s; + for(auto&& item: ie) { + if (!s.empty()) + s += ", "; + s += std::string{item.second.first}; + if (item.first == default_opt) + s += "(*)"; /* default option */ + } + return s; +} + + +/** + * Get map from string->type + */ +template<typename IE> +static std::unordered_map<std::string, typename IE::value_type::first_type> +options_map(const IE& ie) +{ + std::unordered_map<std::string, typename IE::value_type::first_type> map; + for (auto&& item : ie) + map.emplace(std::string{item.second.first}, item.first); + + return map; +} + +// transformers + +// Expand the path using wordexp +static const std::function ExpandPath = [](std::string filepath)->std::string { + if (auto&& res{expand_path(filepath)}; !res) + throw CLI::ValidationError{res.error().what()}; + else + return res.value(); +}; + +// Canonicalize path +static const std::function CanonicalizePath = [](std::string filepath)->std::string { + return filepath = canonicalize_filename(filepath); +}; + +/* + * common + */ + +template<typename T> +static void +sub_crypto(CLI::App& sub, T& opts) +{ + sub.add_flag("--auto-retrieve,-r", opts.auto_retrieve, + "Attempt to automatically retrieve online keys"); + sub.add_flag("--decrypt", opts.decrypt, + "Attempt to decrypt"); +} + +/* + * subcommands + */ + +static void +sub_add(CLI::App& sub, Options& opts) +{ + sub.add_option("files", opts.add.files, + "Path(s) to message files(s)") + ->required(); +} + +static void +sub_cfind(CLI::App& sub, Options& opts) +{ + using Format = Options::Cfind::Format; + static constexpr InfoEnum<Format, 8> FormatInfos = {{ + { Format::Plain, {"plain", "Plain output"} }, + { Format::MuttAlias, {"mutt-alias", "Mutt alias"} }, + { Format::MuttAddressBook, {"mutt-ab", "Mutt address book"}}, + { Format::Wanderlust, {"wl", "Wanderlust"}}, + { Format::OrgContact, {"org-contact", "org-contact"}}, + { Format::Bbdb, {"bbdb", "Emacs BBDB"}}, + { Format::Csv, {"csv", "comma-separated values"}}, + { Format::Json, {"json", "format as json array"}}, + }}; + + const auto fhelp = options_help(FormatInfos, Format::Plain); + const auto fmap = options_map(FormatInfos); + + sub.add_option("--format,-o", opts.cfind.format, + "Output format; one of " + fhelp) + ->type_name("<format>") + ->default_str("plain") + ->default_val(Format::Plain) + ->transform(CLI::CheckedTransformer(fmap)); + + sub.add_option("pattern", opts.cfind.rx_pattern, + "Regular expression pattern to match"); + sub.add_flag("--personal,-p", opts.cfind.personal, + "Only show 'personal' contacts"); + sub.add_option("--after", opts.cfind.after, + "Only show results after some timestamps") + ->type_name("<time_t>") + ->check(CLI::PositiveNumber); + sub.add_option("--maxnum,-n", opts.cfind.maxnum, + "Maximum number of results") + ->type_name("<number>") + ->check(CLI::PositiveNumber); +} + + + +static void +sub_extract(CLI::App& sub, Options& opts) +{ + sub_crypto(sub, opts.extract); + + sub.add_flag("--save-attachments,-a", opts.extract.save_attachments, + "Save all attachments"); + sub.add_flag("--save-all", opts.extract.save_all, "Save all MIME parts") + ->excludes("--save-attachments"); + sub.add_flag("--overwrite", opts.extract.overwrite, + "Overwrite existing files"); + sub.add_flag("--play", opts.extract.play, + "Attempt to open the extracted parts"); + sub.add_option("--parts", opts.extract.parts, + "Save specific parts (comma-sep'd list)") + ->type_name("<parts>")->delimiter(','); + sub.add_option("--target-dir", opts.extract.targetdir, + "Target directory for saving") + ->type_name("<dir>") + ->transform(ExpandPath, "expand target path") + ->default_str("<current>") + ->default_val("."); + sub.add_flag("--uncooked,-u", opts.extract.uncooked, + "Avoid massaging extracted file-names"); + // optional; otherwise use standard-input + sub.add_option("message-path", opts.extract.message, + "Path to message file") + ->type_name("<message-path>"); + + sub.add_option("--matches", opts.extract.filename_rx, + "Regular expression for files to save") + ->type_name("<filename-rx>") + ->excludes("--parts") + ->excludes("--save-attachments") + ->excludes("--save-all"); + + // backward compat: filename-rx as non-option + sub.add_option("filename-rx", opts.extract.filename_rx, + "Regular expression for files to save") + ->type_name("<filename-rx>") + ->excludes("--parts") + ->excludes("--save-attachments") + ->excludes("--matches") + ->excludes("--save-all"); +} + +static void +sub_fields(CLI::App& sub, Options& opts) +{ + // nothing to do. +} + + +static void +sub_find(CLI::App& sub, Options& opts) +{ + using Format = Options::Find::Format; + static constexpr InfoEnum<Format, 7> FormatInfos = {{ + { Format::Plain, + {"plain", "Plain output"} + }, + { Format::Links, + {"links", "Maildir with symbolic links"} + }, + { Format::Xml, + {"xml", "XML"} + }, + { Format::Sexp, + {"sexp", "S-expressions"} + }, + { Format::Json, + {"json", "JSON"} + }, + }}; + + sub.add_flag("--threads,-t", opts.find.threads, + "Show message threads"); + sub.add_flag("--skip-dups,-u", opts.find.skip_dups, + "Show only one of messages with same message-id"); + sub.add_flag("--include-related,-r", opts.find.include_related, + "Include related messages in results"); + sub.add_flag("--analyze,-a", opts.find.analyze, + "Analyze the query"); + + const auto fhelp = options_help(FormatInfos, Format::Plain); + const auto fmap = options_map(FormatInfos); + + sub.add_option("--format,-o", opts.find.format, + "Output format; one of " + fhelp) + ->type_name("<format>") + ->default_str("plain") + ->default_val(Format::Plain) + ->transform(CLI::CheckedTransformer(fmap)); + + sub.add_option("--maxnum,-n", opts.find.maxnum, + "Maximum number of results") + ->type_name("<number>") + ->check(CLI::PositiveNumber); + + sub.add_option("--fields,-f", opts.find.fields, + "Fields to display") + ->default_val("d f s"); + + std::unordered_map<std::string, Field::Id> smap; + std::string sopts; + field_for_each([&](auto&& field){ + if (field.is_sortable()) { + smap.emplace(std::string(field.name), field.id); + smap.emplace(std::string(1, field.shortcut), field.id); + if (!sopts.empty()) + sopts += ", "; + sopts += mu_format("{}|{}", field.name, field.shortcut); + } + }); + sub.add_option("--sortfield,-s", opts.find.sortfield, + "Field to sort the results by; one of " + sopts) + ->type_name("<field>") + ->default_str("date") + ->default_val(Field::Id::Date) + ->transform(CLI::CheckedTransformer(smap)); + + sub.add_flag("--reverse,-z", opts.find.reverse, + "Sort in descending order"); + + sub.add_option("--bookmark,-b", opts.find.bookmark, + "Use bookmarked query") + ->type_name("<bookmark>"); + + sub.add_flag("--clearlinks", opts.find.clearlinks, + "Clear old links first"); + sub.add_option("--linksdir", opts.find.linksdir, + "Use bookmarked query") + ->type_name("<dir>") + ->transform(ExpandPath, "expand linksdir path"); + + sub.add_option("--summary-len", opts.find.summary_len, + "Use up to so many lines for the summary") + ->type_name("<lines>") + ->check(CLI::PositiveNumber); + + sub.add_option("--exec", opts.find.exec, + "Command to execute on message file") + ->type_name("<command>"); + + sub.add_option("query", opts.find.query, + "Search query pattern(s)") + ->type_name("<query>"); +} + +static void +sub_help(CLI::App& sub, Options& opts) +{ + sub.add_option("command", opts.help.command, + "Command to request help for") + ->type_name("<command>"); +} + +static void +sub_index(CLI::App& sub, Options& opts) +{ + sub.add_flag("--lazy-check", opts.index.lazycheck, + "Skip based on dir-timestamps"); + sub.add_flag("--nocleanup", opts.index.nocleanup, + "Don't clean up database after indexing"); + sub.add_flag("--reindex", opts.index.reindex, + "Perform a complete reindexing"); +} + + +static void +sub_info(CLI::App& sub, Options& opts) +{ + sub.add_option("topic", opts.info.topic, + "Information topic") + ->type_name("<topic>") ; +} + +static void +sub_init(CLI::App& sub, Options& opts) +{ + const auto default_mdir = std::invoke([]()->std::string { + if (const auto mdir_env{::getenv("MAILDIR")}; mdir_env) + return mdir_env; + else if (const auto mdir_home = ::join_paths(g_get_home_dir(), "Maildir"); + check_dir(mdir_home)) + return mdir_home; + else + return {}; + }); + + sub.add_option("--maildir,-m", opts.init.maildir, "Top of the maildir") + ->type_name("<maildir>") + ->default_val(default_mdir) + ->transform(ExpandPath, "expand maildir path"); + sub.add_option("--my-address", opts.init.my_addresses, + "Personal e-mail address or regexp") + ->type_name("<address>"); + sub.add_option("--ignored-address", opts.init.ignored_addresses, + "Ignored e-mail address or regexp") + ->type_name("<address>"); + + sub.add_option("--max-message-size", opts.init.max_msg_size, + "Maximum allowed message size in bytes"); + sub.add_option("--batch-size", opts.init.batch_size, + "Maximum size of database transaction"); + sub.add_option("--support-ngrams", opts.init.support_ngrams, + "Support CJK n-grams if for querying/indexing"); + sub.add_flag("--reinit", opts.init.reinit, + "Re-initialize database with current settings") + ->excludes("--maildir") + ->excludes("--my-address") + ->excludes("--ignored-address") + ->excludes("--max-message-size") + ->excludes("--batch-size") + ->excludes("--support-ngrams"); +} + +static void +sub_mkdir(CLI::App& sub, Options& opts) +{ + sub.add_option("--mode", opts.mkdir.mode, "Set the access mode (octal)") + ->default_val(0755) + ->type_name("<mode>"); + + sub.add_option("dirs", opts.mkdir.dirs, "Path to directory/ies") + ->type_name("<dir>") + ->required(); +} + + +static void +sub_move(CLI::App& sub, Options& opts) +{ + sub.add_flag("--change-name", opts.move.change_name, + "Change name of target file"); + sub.add_flag("--update-dups", opts.move.update_dups, + "Update duplicate messages too"); + sub.add_flag("--dry-run,-n", opts.move.dry_run, + "Print target name, but do not change anything"); + + sub.add_option("--flags", opts.move.flags, "Target flags") + ->type_name("<flags>"); + + sub.add_option("source", opts.move.src, "Message file to move") + ->type_name("<message-path>") + ->transform(ExpandPath, "expand source path") + ->transform(CanonicalizePath, "canonicalize source path") + ->required(); + sub.add_option("destination", opts.move.dest, + "Destination maildir") + ->type_name("<maildir>"); +} + + +static void +sub_remove(CLI::App& sub, Options& opts) +{ + sub.add_option("files", opts.remove.files, + "Paths to message files to remove") + ->type_name("<files>"); +} + +static void +sub_server(CLI::App& sub, Options& opts) +{ + sub.add_flag("--commands", opts.server.commands, + "List available commands"); + sub.add_option("--eval", opts.server.eval, + "Evaluate mu server expression") + ->excludes("--commands"); + sub.add_flag("--allow-temp-file", opts.server.allow_temp_file, + "Allow for the temp-file optimization") + ->excludes("--commands"); + +} + +static void +sub_verify(CLI::App& sub, Options& opts) +{ + sub_crypto(sub, opts.verify); + + // optional; otherwise use standard-input + sub.add_option("message-paths", opts.verify.files, + "Message files to verify") + ->type_name("<message-path>"); +} + +static void +sub_view(CLI::App& sub, Options& opts) +{ + using Format = Options::View::Format; + static constexpr InfoEnum<Format, 3> FormatInfos = {{ + { Format::Plain, + {"plain", "Plain output"} + }, + { Format::Html, + {"html", "Plain output with HTML body"} + }, + { Format::Sexp, + {"sexp", "S-expressions"} + }, + }}; + + const auto fhelp = options_help(FormatInfos, Format::Plain); + const auto fmap = options_map(FormatInfos); + + sub.add_option("--format,-o", opts.view.format, + "Output format; one of " + fhelp) + ->type_name("<format>") + ->default_str("plain") + ->default_val(Format::Plain) + ->transform(CLI::CheckedTransformer(fmap)); + + sub_crypto(sub, opts.view); + + sub.add_option("--summary-len", opts.view.summary_len, + "Use up to so many lines for the summary") + ->type_name("<lines>") + ->check(CLI::PositiveNumber); + + sub.add_flag("--terminate", opts.view.terminate, + "Insert form-feed after each message"); + + // optional; otherwise use standard-input + sub.add_option("message-paths", opts.view.files, + "Message files to view") + ->type_name("<message-path>"); +} + + +using SubCommand = Options::SubCommand; +using Category = Options::Category; + +struct CommandInfo { + Category category; + std::string_view name; + std::string_view help; + + // std::function is not constexp-friendly + typedef void(*setup_func_t)(CLI::App&, Options&); + setup_func_t setup_func{}; +}; + +static constexpr +AssocPairs<SubCommand, CommandInfo, Options::SubCommandNum> SubCommandInfos= {{ + { SubCommand::Add, + { Category::NeedsWritableStore, + "add", "Add message(s) to the database", sub_add} + }, + { SubCommand::Cfind, + { Category::NeedsReadOnlyStore, + "cfind", "Find contacts matching pattern", sub_cfind} + }, + { SubCommand::Extract, + {Category::None, + "extract", "Extract MIME-parts from messages", sub_extract} + }, + { SubCommand::Fields, + {Category::None, + "fields", "Superseded by 'mu info'", sub_fields} + }, + { SubCommand::Find, + {Category::NeedsReadOnlyStore, + "find", "Find messages matching query", sub_find } + }, + { SubCommand::Help, + {Category::None, + "help", "Show help information", sub_help } + }, + { SubCommand::Index, + {Category::NeedsWritableStore, + "index", "Store message information in the database", sub_index } + }, + { SubCommand::Info, + {Category::NeedsReadOnlyStore, + "info", "Show information", sub_info } + }, + { SubCommand::Init, + {Category::NeedsWritableStore, + "init", "Initialize the database", sub_init } + }, + { SubCommand::Mkdir, + {Category::None, + "mkdir", "Create a new Maildir", sub_mkdir } + }, + { SubCommand::Move, + {Category::NeedsWritableStore, + "move", "Move a message or change flags", sub_move } + }, + { SubCommand::Remove, + {Category::NeedsWritableStore, + "remove", "Remove message from file-system and database", sub_remove } + }, + { SubCommand::Script, + // Note: SubCommand::Script is special; there's no literal + // "script" subcommand, there subcommands for all the scripts. + {Category::None, + "script", "Invoke a script", {}} + }, + { SubCommand::Server, + {Category::NeedsWritableStore, + "server", "Start a mu server (for mu4e)", sub_server} + }, + { SubCommand::Verify, + {Category::None, + "verify", "Verify cryptographic signatures", sub_verify} + }, + { SubCommand::View, + {Category::None, + "view", "View specific messages", sub_view} + }, + }}; + + + +static ScriptInfos +add_scripts(CLI::App& app, Options& opts) +{ +#ifndef BUILD_GUILE + return {}; +#else + ScriptPaths paths = { MU_SCRIPTS_DIR }; + auto scriptinfos{script_infos(paths)}; + for (auto&& script: scriptinfos) { + auto&& sub = app.add_subcommand(script.name)->group("Scripts") + ->description(script.oneline); + sub->add_option("params", opts.script.params, + "Parameter to script") + ->type_name("<params>"); + } + + return scriptinfos; +#endif /*BUILD_GUILE*/ +} + + +static Result<Options> +show_manpage(Options& opts, const std::string& name) +{ + char *path = g_find_program_in_path("man"); + if (!path) + return Err(Error::Code::Command, + "cannot find 'man' program"); + + GError* err{}; + auto cmd{to_string_gchar(std::move(path)) + " " + name}; + auto res = g_spawn_command_line_sync(cmd.c_str(), {}, {}, {}, &err); + if (!res) + return Err(Error::Code::Command, &err, + "error running man command"); + + return Ok(std::move(opts)); +} + + +static Result<Options> +cmd_help(const CLI::App& app, Options& opts) +{ + if (opts.help.command.empty()) { + mu_println("{}", app.help()); + return Ok(std::move(opts)); + } + + for (auto&& item: SubCommandInfos) { + if (item.second.name == opts.help.command) + return show_manpage(opts, "mu-" + opts.help.command); + } + + for (auto&& item: {"query", "easy"}) + if (item == opts.help.command) + return show_manpage(opts, "mu-" + opts.help.command); + + return Err(Error::Code::Command, + "no help available for '{}'", opts.help.command); +} + +bool +Options::default_no_color() +{ + static const auto no_color = + !::isatty(::fileno(stdout)) || + !::isatty(::fileno(stderr)) || + ::getenv("NO_COLOR") != NULL; + + return no_color; +} + +static void +add_global_options(CLI::App& cli, Options& opts) +{ + opts.nocolor = Options::default_no_color(); + errno = 0; + + cli.add_flag("-q,--quiet", opts.quiet, "Hide non-essential output"); + cli.add_flag("-v,--verbose", opts.verbose, "Show verbose output"); + cli.add_flag("--log-stderr", opts.log_stderr, "Log to stderr") + ->group(""/*always hide*/); + cli.add_flag("--nocolor", opts.nocolor, "Don't show ANSI colors") + ->default_val(Options::default_no_color()) + ->default_str(Options::default_no_color() ? "<true>" : "<false>"); + cli.add_flag("-d,--debug", opts.debug, "Run in debug mode") + ->group(""/*always hide*/); +} + +Result<Options> +Options::make(int argc, char *argv[]) +{ + Options opts{}; + CLI::App app{"mu mail indexer/searcher", "mu"}; + + app.description(R"(mu mail indexer/searcher +Copyright (C) 2008-2023 Dirk-Jan C. Binnema + +License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>. +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +)"); + app.set_version_flag("-V,--version", PACKAGE_VERSION); + app.set_help_flag("-h,--help", "Show help informmation"); + app.set_help_all_flag("--help-all"); + app.require_subcommand(0, 1); + + add_global_options(app, opts); + + /* + * subcommands + * + * we keep around a map of the subcommand pointers, so we can + * easily find the chosen one (if any) later. + */ + for (auto&& cmdinfo: SubCommandInfos) { + //const auto cmdtype = cmdinfo.first; + const auto name{std::string{cmdinfo.second.name}}; + const auto help{std::string{cmdinfo.second.help}}; + const auto setup{cmdinfo.second.setup_func}; + const auto cat{category(cmdinfo.first)}; + + if (!setup) + continue; + + auto sub = app.add_subcommand(name, help); + setup(*sub, opts); + + /* allow global options _after_ subcommand as well; + * this is for backward compat with the older + * command-line parsing */ + sub->fallthrough(true); + + /* store commands get the '--muhome' parameter as well */ + if (cat == Category::NeedsReadOnlyStore || + cat == Category::NeedsWritableStore) + sub->add_option("--muhome", + opts.muhome, "Specify alternative mu directory") + ->envname("MUHOME") + ->type_name("<dir>") + ->transform(ExpandPath, "expand muhome path"); + } + + /* add scripts (if supported) as semi-subcommands as well */ + const auto scripts = add_scripts(app, opts); + + try { + app.parse(argc, argv); + + // find the chosen sub command, if any. + for (auto&& cmdinfo: SubCommandInfos) { + if (cmdinfo.first == SubCommand::Script) + continue; // not a _real_ subcommand. + const auto name{std::string{cmdinfo.second.name}}; + if (app.got_subcommand(name)) { + opts.sub_command = cmdinfo.first; + } + } + + // otherwise, perhaps it's a script? + if (!opts.sub_command) { + for (auto&& info: scripts) { // find the chosen script, if any. + if (app.got_subcommand(info.name)) { + opts.sub_command = SubCommand::Script; + opts.script.name = info.name; + } + } + } + + // if nothing else, try "help" + if (opts.sub_command.value_or(SubCommand::Help) == SubCommand::Help) + return cmd_help(app, opts); + + } catch (const CLI::CallForHelp& cfh) { + mu_println("{}", app.help()); + } catch (const CLI::CallForAllHelp& cfah) { + mu_println("{}", app.help("", CLI::AppFormatMode::All)); + } catch (const CLI::CallForVersion&) { + mu_println("version {}", PACKAGE_VERSION); + } catch (const CLI::ParseError& pe) { + return Err(Error::Code::InvalidArgument, "{}", pe.what()); + } catch (...) { + return Err(Error::Code::Internal, "error parsing arguments"); + } + + return Ok(std::move(opts)); +} + +Category +Options::category(Options::SubCommand sub) +{ + for (auto&& item: SubCommandInfos) + if (item.first == sub) + return item.second.category; + + return Category::None; +} + +/* + * trust but verify + */ + +static constexpr bool +validate_subcommand_ids() +{ + size_t val{}; + for (auto& cmd: Options::SubCommands) + if (static_cast<size_t>(cmd) != val++) + return false; + + for (auto u = 0U; u != SubCommandInfos.size(); ++u) + if (static_cast<size_t>(SubCommandInfos.at(u).first) != u) + return false; + return true; +} + + +/* + * tests... also build as runtime-tests, so we can get coverage info + */ +#ifdef BUILD_TESTS +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + + +[[maybe_unused]] +static void +test_ids() +{ + static_assert(validate_subcommand_ids()); +} + +#ifdef BUILD_TESTS + +enum struct TestEnum { A, B, C }; +constexpr AssocPairs<TestEnum, std::string_view, 3> +test_epairs = {{ + {TestEnum::A, "a"}, + {TestEnum::B, "b"}, + {TestEnum::C, "c"}, +}}; + +static constexpr Option<std::string_view> +to_name(TestEnum te) +{ + return to_second(test_epairs, te); +} + +static constexpr Option<TestEnum> +to_type(std::string_view name) +{ + return to_first(test_epairs, name); + +} + +static void +test_enum_pairs(void) +{ + assert_equal(to_name(TestEnum::A).value(), "a"); + g_assert_true(to_type("c").value() == TestEnum::C); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/options/ids", test_ids); + g_test_add_func("/option/enum-pairs", test_enum_pairs); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-options.hh b/mu/mu-options.hh new file mode 100644 index 0000000..fa440bf --- /dev/null +++ b/mu/mu-options.hh @@ -0,0 +1,310 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_OPTIONS_HH__ +#define MU_OPTIONS_HH__ + +#include <sstream> +#include <string> +#include <vector> +#include <utils/mu-option.hh> +#include <utils/mu-result.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-utils-file.hh> + +#include <message/mu-fields.hh> +#include <mu-script.hh> +#include <ctime> +#include <sys/stat.h> + +/* command-line options for Mu */ +namespace Mu { +struct Options { + using OptSize = Option<std::size_t>; + using SizeVec = std::vector<std::size_t>; + using OptTStamp = Option<std::time_t>; + using OptFieldId = Option<Field::Id>; + using StringVec = std::vector<std::string>; + + /* + * general options + */ + bool quiet; /**< don't give any output */ + bool debug; /**< log debug-level info */ + bool version; /**< request mu version */ + bool log_stderr; /**< log to stderr */ + bool nocolor; /**< don't use use ansi-colors */ + bool verbose; /**< verbose output */ + std::string muhome; /**< alternative mu dir */ + + /** + * Whether by default, we should show color + * + * @return true or false + */ + static bool default_no_color(); + + enum struct SubCommand { + Add, Cfind, Extract, Fields, Find, Help, Index,Info, Init, Mkdir, + Move, Remove, Script, Server, Verify, View, + // <private> + __count__ + }; + static constexpr auto SubCommandNum = static_cast<size_t>(SubCommand::__count__); + static constexpr std::array<SubCommand, SubCommandNum> SubCommands = {{ + SubCommand::Add, + SubCommand::Cfind, + SubCommand::Extract, + SubCommand::Fields, + SubCommand::Find, + SubCommand::Help, + SubCommand::Index, + SubCommand::Info, + SubCommand::Init, + SubCommand::Mkdir, + SubCommand::Move, + SubCommand::Remove, + SubCommand::Script, + SubCommand::Server, + SubCommand::Verify, + SubCommand::View + }}; + + Option<SubCommand> sub_command; /**< The chosen sub-command, if any. */ + + /* + * Add + */ + struct Add { + StringVec files; /**< field to add */ + } add; + + /* + * Cfind + */ + struct Cfind { + enum struct Format { Plain, MuttAlias, MuttAddressBook, + Wanderlust, OrgContact, Bbdb, Csv, Json }; + Format format; /**< Output format */ + bool personal; /**< only show personal contacts */ + OptTStamp after; /**< only last seen after tstamp */ + OptSize maxnum; /**< maximum number of results */ + std::string rx_pattern; /**< contact regexp to match */ + } cfind; + + + struct Crypto { + bool auto_retrieve; /**< auto-retrieve keys */ + bool decrypt; /**< decrypt */ + }; + + /* + * Extract + */ + struct Extract: public Crypto { + std::string message; /**< path to message file */ + bool save_all; /**< extract all parts */ + bool save_attachments; /**< extract all attachment parts */ + SizeVec parts; /**< parts to save / open */ + std::string targetdir{}; /**< where to save attachments */ + bool overwrite; /**< overwrite same-named files */ + bool play; /**< try to 'play' attachment */ + std::string filename_rx; /**< Filename rx to save */ + bool uncooked{}; /**< Whether to avoid massaging + * the output filename */ + } extract; + + /* + * Fields + */ + + /* + * Find + */ + struct Find { + std::string fields; /**< fields to show in output */ + Field::Id sortfield; /**< field to sort by */ + OptSize maxnum; /**< max # of entries to print */ + bool reverse; /**< sort in revers order (z->a) */ + bool threads; /**< show message threads */ + bool clearlinks; /**< clear linksdir first */ + std::string linksdir; /**< directory for links */ + OptSize summary_len; /**< max # of lines for summary */ + std::string bookmark; /**< use bookmark */ + bool analyze; /**< analyze query */ + + enum struct Format { Plain, Links, Xml, Json, Sexp, Exec }; + Format format; /**< Output format */ + std::string exec; /**< cmd to execute on matches */ + bool skip_dups; /**< show only first with msg id */ + bool include_related; /**< included related messages */ + /**< for find and cind */ + OptTStamp after; /**< only last seen after T */ + bool auto_retrieve; /**< assume we're online */ + bool decrypt; /**< try to decrypt the body */ + + StringVec query; /**< search query */ + } find; + + struct Help { + std::string command; /**< Help parameter */ + } help; + + /* + * Index + */ + struct Index { + bool nocleanup; /**< don't cleanup del'd mails */ + bool lazycheck; /**< don't check uptodate dirs */ + bool reindex; /**< do a full re-index */ + } index; + + + /* + * Info + */ + struct Info { + std::string topic; /**< what to get info about? */ + } info; + + /* + * Init + */ + struct Init { + std::string maildir; /**< where the mails are */ + StringVec my_addresses; /**< personal e-mail addresses */ + StringVec ignored_addresses; /**< addresses to be ignored for + * the contacts-cache */ + OptSize max_msg_size; /**< max size for message files */ + OptSize batch_size; /**< db transaction batch size */ + bool reinit; /**< re-initialize */ + bool support_ngrams; /**< support CJK etc. ngrams */ + + } init; + + /* + * Mkdir + */ + struct Mkdir { + StringVec dirs; /**< Dir(s) to create */ + mode_t mode; /**< Mode for the maildir */ + } mkdir; + + /* + * Move + */ + struct Move { + std::string src; /**< Source file */ + std::string dest; /**< Destination dir */ + std::string flags; /**< Flags for destination */ + bool change_name; /**< Change basename for destination */ + bool update_dups; /**< Update duplicate messages too */ + bool dry_run; /**< Just print the result path, + but do not change anything */ + } move; + + /* + * Remove + */ + struct Remove { + StringVec files; /**< Files to remove */ + } remove; + + /* + * Scripts (i.e., finding scriot) + */ + struct Script { + std::string name; /**< name of script */ + StringVec params; /**< script params */ + } script; + + /* + * Server + */ + struct Server { + bool commands; /**< dump docs for commands */ + std::string eval; /**< command to evaluate */ + bool allow_temp_file; /**< temp-file optimization allowed? */ + } server; + + /* + * Verify + */ + struct Verify: public Crypto { + StringVec files; /**< message files to verify */ + } verify; + /* + * View + */ + struct View: public Crypto { + bool terminate; /**< add \f between msgs in view */ + OptSize summary_len; /**< max # of lines for summary */ + + enum struct Format { Plain, Sexp, Html }; + Format format; /**< output format*/ + + StringVec files; /**< Message file(s) */ + } view; + + + /** + * Create an Options structure fo the given command-line arguments. + * + * @param argc argc + * @param argv argc + * + * @return Options, or an Error + */ + static Result<Options> make(int argc, char *argv[]); + + + /** + * Different commands need different things + * + */ + enum struct Category { + None, + NeedsReadOnlyStore, + NeedsWritableStore, + }; + + /** + * Get the category for some subcommand + * + * @param sub subcommand + * + * @return the category + */ + static Category category(SubCommand sub); + + /** + * Get some well-known Path + * + * @param path the Path to find + * + * @return the path name + */ + std::string runtime_path(RuntimePath path) const { + return Mu::runtime_path(path, muhome); + } +}; + +} // namepace Mu + +#endif /* MU_OPTIONS_HH__ */ diff --git a/mu/mu.cc b/mu/mu.cc new file mode 100644 index 0000000..69d3c48 --- /dev/null +++ b/mu/mu.cc @@ -0,0 +1,136 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include <config.h> +#include <functional> + +#include <glib.h> +#include <glib-object.h> +#include <locale.h> + +#include "mu-cmd.hh" +#include "mu-options.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-logger.hh" + +#include "mu-cmd.hh" + +using namespace Mu; + + +static void +output_error(const std::string& what, bool use_color) +{ + using Color = MaybeAnsi::Color; + MaybeAnsi col{use_color}; + + mu_printerrln("{}error{}: {}{}{}", + col.fg(Color::Red), col.reset(), + col.fg(Color::BrightYellow), what, col.reset()); +} + +static int +handle_result(const Result<void>& res, const Mu::Options& opts) +{ + if (res) + return 0; + + using Color = MaybeAnsi::Color; + MaybeAnsi col{!opts.nocolor}; + + // show the error and some help, but not if it's only a softerror. + if (!res.error().is_soft_error()) + output_error(res.error().what(), !opts.nocolor); + else + mu_printerrln("{}{}{}", + col.fg(Color::BrightBlue), res.error().what(), col.reset()); + + // perhaps give some useful hint on how to solve it. + if (!res.error().hint().empty()) + mu_printerrln("{}hint{}: {}{}{}", + col.fg(Color::Blue), col.reset(), + col.fg(Color::Green), res.error().hint(), col.reset()); + + if (res.error().exit_code() != 0 && !res.error().is_soft_error()) { + mu_warning("mu finishing with error: {}", + format_as(res.error())); + if (const auto& hint = res.error().hint(); !hint.empty()) + mu_info("hint: {}", hint); + } + + return res.error().exit_code(); +} + +int +main(int argc, char* argv[]) try +{ + /* + * We handle this through explicit options + */ + g_unsetenv("XAPIAN_CJK_NGRAM"); + + /* + * set up locale + */ + ::setlocale(LC_ALL, ""); + + /* + * read command-line options + */ + const auto opts{Options::make(argc, argv)}; + if (!opts) { + output_error(opts.error().what(), !Options::default_no_color()); + return opts.error().exit_code(); + } else if (!opts->sub_command) { + // nothing more to do. + return 0; + } + + // setup logging + Logger::Options lopts{Logger::Options::None}; + if (opts->log_stderr) + lopts |= Logger::Options::StdOutErr; + if (opts->debug) + lopts |= Logger::Options::Debug; + if (!!g_getenv("MU_TEST")) + lopts |= Logger::Options::File; + + const auto logger{Logger::make(opts->runtime_path(RuntimePath::LogFile), lopts)}; + if (!logger) { + output_error(logger.error().what(), !opts->nocolor); + return logger.error().exit_code(); + } + + /* + * handle sub command + */ + return handle_result(mu_cmd_execute(*opts), *opts); + + // exceptions should have been handled earlier, but catch them here, + // just in case... +} catch (const std::logic_error& le) { + mu_printerrln("caught logic-error: {}", le.what()); + return 97; +} catch (const std::runtime_error& re) { + mu_printerrln("caught runtime-error: {}", re.what()); + return 98; +} catch (...) { + mu_printerrln("caught exception"); + return 99; +} diff --git a/mu/tests/gmime-test.c b/mu/tests/gmime-test.c new file mode 100644 index 0000000..f269ecb --- /dev/null +++ b/mu/tests/gmime-test.c @@ -0,0 +1,264 @@ +/* +** Copyright (C) 2011-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#define _POSIX_C_SOURCE 1 + +#include <gmime/gmime.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <locale.h> + +static gchar* +get_recip(GMimeMessage* msg, GMimeAddressType atype) +{ + char* recep; + InternetAddressList* receps; + + receps = g_mime_message_get_addresses(msg, atype); + recep = (char*)internet_address_list_to_string(receps, NULL, FALSE); + + if (!recep || !*recep) { + g_free(recep); + return NULL; + } + + return recep; +} + +static gchar* +get_refs_str(GMimeMessage* msg) +{ + const gchar* str; + GMimeReferences* mime_refs; + int i, refs_len; + gchar* rv; + + str = g_mime_object_get_header(GMIME_OBJECT(msg), "References"); + if (!str) + return NULL; + + mime_refs = g_mime_references_parse(NULL, str); + refs_len = g_mime_references_length(mime_refs); + for (rv = NULL, i = 0; i < refs_len; ++i) { + const char* msgid; + char *tmp; + + msgid = g_mime_references_get_message_id(mime_refs, i); + tmp = rv; + rv = g_strdup_printf("%s%s%s", rv ? rv : "", rv ? "," : "", msgid); + g_free(tmp); + } + g_mime_references_free(mime_refs); + + return rv; +} + +static void +print_date(GMimeMessage* msg) +{ + GDateTime* dt; + gchar* buf; + + dt = g_mime_message_get_date(msg); + if (!dt) + return; + + dt = g_date_time_to_local(dt); + buf = g_date_time_format(dt, "%c"); + g_date_time_unref(dt); + + if (buf) { + g_print("Date : %s\n", buf); + g_free(buf); + } +} + +static void +print_body(GMimeMessage* msg) +{ + GMimeObject* body; + GMimeDataWrapper* wrapper; + GMimeStream* stream; + + body = g_mime_message_get_body(msg); + + if (GMIME_IS_MULTIPART(body)) + body = g_mime_multipart_get_part(GMIME_MULTIPART(body), 0); + if (!GMIME_IS_PART(body)) + return; + + wrapper = g_mime_part_get_content(GMIME_PART(body)); + if (!GMIME_IS_DATA_WRAPPER(wrapper)) + return; + + stream = g_mime_data_wrapper_get_stream(wrapper); + if (!GMIME_IS_STREAM(stream)) + return; + + do { + char buf[512]; + ssize_t len; + + len = g_mime_stream_read(stream, buf, sizeof(buf)); + if (len == -1) + break; + + if (write(fileno(stdout), buf, len) == -1) + break; + + if (len < (int)sizeof(buf)) + break; + + } while (1); +} + +static gboolean +test_message(GMimeMessage* msg) +{ + gchar* val; + const gchar* str; + + val = get_recip(msg, GMIME_ADDRESS_TYPE_FROM); + g_print("From : %s\n", val ? val : "<none>"); + g_free(val); + + val = get_recip(msg, GMIME_ADDRESS_TYPE_TO); + g_print("To : %s\n", val ? val : "<none>"); + g_free(val); + + val = get_recip(msg, GMIME_ADDRESS_TYPE_CC); + g_print("Cc : %s\n", val ? val : "<none>"); + g_free(val); + + val = get_recip(msg, GMIME_ADDRESS_TYPE_BCC); + g_print("Bcc : %s\n", val ? val : "<none>"); + g_free(val); + + str = g_mime_message_get_subject(msg); + g_print("Subject: %s\n", str ? str : "<none>"); + + print_date(msg); + + str = g_mime_message_get_message_id(msg); + g_print("Msg-id : %s\n", str ? str : "<none>"); + + { + gchar* refsstr; + refsstr = get_refs_str(msg); + g_print("Refs : %s\n", refsstr ? refsstr : "<none>"); + g_free(refsstr); + } + + print_body(msg); + + return TRUE; +} + +static gboolean +test_stream(GMimeStream* stream) +{ + GMimeParser* parser; + GMimeMessage* msg; + gboolean rv; + + parser = NULL; + msg = NULL; + + parser = g_mime_parser_new_with_stream(stream); + if (!parser) { + g_warning("failed to create parser"); + rv = FALSE; + goto leave; + } + + msg = g_mime_parser_construct_message(parser, NULL); + if (!msg) { + g_warning("failed to construct message"); + rv = FALSE; + goto leave; + } + + rv = test_message(msg); + +leave: + if (parser) + g_object_unref(parser); + + if (msg) + g_object_unref(msg); + + return rv; +} + +static gboolean +test_file(const char* path) +{ + FILE* file; + GMimeStream* stream; + gboolean rv; + + stream = NULL; + file = NULL; + + file = fopen(path, "r"); + if (!file) { + g_warning("cannot open file '%s': %s", path, g_strerror(errno)); + rv = FALSE; + goto leave; + } + + stream = g_mime_stream_file_new(file); + if (!stream) { + g_warning("cannot open stream for '%s'", path); + rv = FALSE; + goto leave; + } + + rv = test_stream(stream); + g_object_unref(stream); + return rv; + +leave: + if (file) + fclose(file); + + return rv; +} + +int +main(int argc, char* argv[]) +{ + gboolean rv; + + if (argc != 2) { + g_printerr("usage: %s <msg-file>\n", argv[0]); + return 1; + } + + setlocale(LC_ALL, ""); + + g_mime_init(); + + rv = test_file(argv[1]); + + g_mime_shutdown(); + + return rv ? 0 : 1; +} diff --git a/mu/tests/meson.build b/mu/tests/meson.build new file mode 100644 index 0000000..cc8a342 --- /dev/null +++ b/mu/tests/meson.build @@ -0,0 +1,110 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# tests +# + + +test('test-cmd-add', + executable('test-cmd-add', + '../mu-cmd-add.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-cfind', + executable('test-cmd-cfind', + '../mu-cmd-cfind.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-extract', + executable('test-cmd-extract', + '../mu-cmd-extract.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-find', + executable('test-cmd-find', + '../mu-cmd-find.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-index', + executable('test-cmd-index', + '../mu-cmd-index.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-init', + executable('test-cmd-init', + '../mu-cmd-init.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-mkdir', + executable('test-cmd-mkdir', + '../mu-cmd-mkdir.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-move', + executable('test-cmd-move', + '../mu-cmd-move.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-remove', + executable('test-cmd-remove', + '../mu-cmd-remove.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-verify', + executable('test-cmd-verify', + '../mu-cmd-verify.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-view', + executable('test-cmd-view', + '../mu-cmd-view.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-query', + executable('test-cmd-query', + 'test-mu-query.cc', + install: false, + dependencies: [glib_dep, config_h_dep, lib_mu_dep])) + +gmime_test = executable( + 'gmime-test', [ + 'gmime-test.c' +], + dependencies: [ glib_dep, gmime_dep ], + install: false) diff --git a/mu/tests/test-mu-query.cc b/mu/tests/test-mu-query.cc new file mode 100644 index 0000000..09c0cde --- /dev/null +++ b/mu/tests/test-mu-query.cc @@ -0,0 +1,612 @@ +/* +** Copyright (C) 2008-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <unordered_set> +#include <string> + +#include <glib.h> +#include <glib/gstdio.h> + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <locale.h> + +#include "utils/mu-test-utils.hh" +#include "mu-query.hh" +#include "utils/mu-result.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include "mu-store.hh" + +using namespace Mu; + +static std::string DB_PATH1; +static std::string DB_PATH2; + +static std::string +make_database(const std::string& dbdir, const std::string& testdir) +{ + /* use the env var rather than `--muhome` */ + g_setenv("MUHOME", dbdir.c_str(), 1); + const auto cmdline{mu_format( + "/bin/sh -c '" + "{} --quiet init --maildir={} ; " + "{} --quiet index'", + MU_PROGRAM, testdir, MU_PROGRAM)}; + + if (g_test_verbose()) + mu_printerrln("\n{}", cmdline); + + g_assert(g_spawn_command_line_sync(cmdline.c_str(), NULL, NULL, NULL, NULL)); + auto xpath = join_paths(dbdir, "xapian"); + /* ensure MUHOME worked */ + g_assert_cmpuint(::access(xpath.c_str(), F_OK), ==, 0); + + return xpath; +} + +static void +assert_no_dups(const QueryResults& qres) +{ + std::unordered_set<std::string> msgid_set, path_set; + + for (auto&& mi : qres) { + g_assert_true(msgid_set.find(mi.message_id().value()) == msgid_set.end()); + g_assert_true(path_set.find(mi.path().value()) == path_set.end()); + + path_set.emplace(*mi.path()); + msgid_set.emplace(*mi.message_id()); + + g_assert_false(msgid_set.find(mi.message_id().value()) == msgid_set.end()); + g_assert_false(path_set.find(mi.path().value()) == path_set.end()); + } +} + +/* note: this also *moves the iter* */ +static size_t +run_and_count_matches(const std::string& xpath, + const std::string& expr, + Mu::QueryFlags flags = Mu::QueryFlags::None) +{ + auto store{Store::make(xpath)}; + assert_valid_result(store); + + // if (g_test_verbose()) { + // std::cout << "==> mquery: " << store.parse_query(expr, false) << "\n"; + // std::cout << "==> xquery: " << store.parse_query(expr, true) << "\n"; + // } + + Mu::allow_warnings(); + + auto qres{store->run_query(expr, {}, flags)}; + g_assert_true(!!qres); + assert_no_dups(*qres); + + if (g_test_verbose()) + mu_println("'{}' => {}\n", expr, qres->size()); + + return qres->size(); +} + +typedef struct { + const char* query; + size_t count; /* expected number of matches */ +} QResults; + +static void +test_mu_query_01(void) +{ + int i; + QResults queries[] = { + {"basic", 3}, + {"question", 5}, + {"thanks", 2}, + {"html", 4}, + {"subject:exception", 1}, + {"exception", 1}, + {"subject:A&B", 1}, + {"A&B", 1}, + {"subject:elisp", 1}, + {"html AND contains", 1}, + {"html and contains", 1}, + {"from:pepernoot", 0}, + {"foo:pepernoot", 0}, + {"funky", 1}, + {"fünkÿ", 1}, + { "", 19 }, + {"msgid:abcd$efgh@example.com", 1}, + {"i:abcd$efgh@example.com", 1}, +#ifdef HAVE_CLD2 +{ "lang:en", 14}, +#endif /*HAVE_CLD2*/ + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_02(void) +{ + const char* q; + q = "i:f7ccd24b0808061357t453f5962w8b61f9a453b684d0@mail.gmail.com"; + g_assert_cmpuint(run_and_count_matches(DB_PATH1, q), ==, 1); +} + +static void +test_mu_query_03(void) +{ + int i; + QResults queries[] = {{"ploughed", 1}, + {"i:3BE9E6535E3029448670913581E7A1A20D852173@" + "emss35m06.us.lmco.com", + 1}, + {"i:!&!AAAAAAAAAYAAAAAAAAAOH1+8mkk+lLn7Gg5fke7" + "FbCgAAAEAAAAJ7eBDgcactKhXL6r8cEnJ8BAAAAAA==@" + "example.com", + 1}, + + /* subsets of the words in the subject should match */ + {"s:gcc include search order", 1}, + {"s:gcc include search", 1}, + {"s:search order", 1}, + {"s:include", 1}, + + {"s:lisp", 1}, + {"s:LISP", 1}, + + // { "s:\"Re: Learning LISP; Scheme vs elisp.\"", 1}, + // { "subject:Re: Learning LISP; Scheme vs elisp.", 1}, + // { "subject:\"Re: Learning LISP; Scheme vs elisp.\"", 1}, + {"to:help-gnu-emacs@gnu.org", 4}, + //{"t:help-gnu-emacs", 4}, + {"flag:flagged", 1}}; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_04(void) +{ + int i; + + QResults queries[] = { + {"frodo@example.com", 1}, + {"f:frodo@example.com", 1}, + {"f:Frodo Baggins", 1}, + {"bilbo@anotherexample.com", 1}, + {"t:bilbo@anotherexample.com", 1}, + {"t:bilbo", 1}, + {"f:bilbo", 0}, + {"baggins", 1}, + {"prio:h", 1}, + {"prio:high", 1}, + {"prio:normal", 11}, + {"prio:l", 7}, + {"not prio:l", 12}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_logic(void) +{ + int i; + QResults queries[] = {{"subject:gcc", 1}, + {"subject:lisp", 1}, + {"subject:gcc OR subject:lisp", 2}, + {"subject:gcc or subject:lisp", 2}, + {"subject:gcc AND subject:lisp", 0}, + {"subject:gcc OR (subject:scheme AND subject:elisp)", 2}, + {"(subject:gcc OR subject:scheme) AND subject:elisp", 1}}; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_accented_chars_01(void) +{ + auto store = Store::make(DB_PATH1); + assert_valid_result(store); + + auto qres{store->run_query("fünkÿ")}; + g_assert_true(!!qres); + g_assert_false(qres->empty()); + + const auto msg{qres->begin().message()}; + if (!msg) { + mu_warning("error getting message"); + g_assert_not_reached(); + } + + assert_equal(msg->subject(), "Greetings from Lothlórien"); +} + +static void +test_mu_query_accented_chars_02(void) +{ + int i; + + QResults queries[] = {{"f:mü", 1}, + { "s:motörhead", 1}, + {"t:Helmut", 1}, + {"t:Kröger", 1}, + {"s:MotorHeäD", 1}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + auto count = run_and_count_matches(DB_PATH1, queries[i].query); + if (count != queries[i].count) + mu_warning("query '{}'; expected {} but got {}", + queries[i].query, queries[i].count, count); + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); + } +} + +static void +test_mu_query_accented_chars_fraiche(void) +{ + int i; + + QResults queries[] = {{"crème fraîche", 1}, + {"creme fraiche", 1}, + {"fraîche crème", 1}, + {"будланула", 1}, + {"БУДЛАНУЛА", 1}, + {"CRÈME FRAÎCHE", 1}, + {"CREME FRAICHE", 1}}; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + if (g_test_verbose()) + mu_println("{}", queries[i].query); + + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); + } +} + +static void +test_mu_query_wildcards(void) +{ + int i; + + QResults queries[] = { + {"f:mü", 1}, + {"s:mo*", 1}, + {"t:Helm*", 1}, + {"queensryche", 1}, + {"Queen*", 1}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_dates_helsinki(void) +{ + const auto hki = "Europe/Helsinki"; + if (!timezone_available(hki)) { + g_test_skip("timezone not available"); + return; + } + + int i; + const char* old_tz; + + QResults queries[] = {{"date:20080731..20080804", 5}, + {"date:20080731..20080804 s:gcc", 1}, + {"date:200808110803..now", 7}, + {"date:200808110803..today", 7}, + {"date:200808110801..now", 7}}; + + old_tz = set_tz(hki); + TempDir tdir; + const auto xpath{make_database(tdir.path(), MU_TESTMAILDIR)}; + g_assert_false(xpath.empty()); + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(xpath, queries[i].query), + ==, queries[i].count); + + set_tz(old_tz); +} + +static void +test_mu_query_dates_sydney(void) +{ + const auto syd = "Australia/Sydney"; + if (!timezone_available(syd)) { + g_test_skip("timezone not available"); + return; + } + + int i; + const char* old_tz; + QResults queries[] = {{"date:20080731..20080804", 5}, + {"date:20080731..20080804 s:gcc", 1}, + {"date:200808110803..now", 7}, + {"date:200808110803..today", 7}, + {"date:200808110801..now", 7}}; + old_tz = set_tz(syd); + + TempDir tdir; + const auto xpath{make_database(tdir.path(), MU_TESTMAILDIR)}; + g_assert_false(xpath.empty()); + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(xpath, queries[i].query), + ==, queries[i].count); + set_tz(old_tz); +} + +static void +test_mu_query_dates_la(void) +{ + const auto la = "America/Los_Angeles"; + if (!timezone_available(la)) { + g_test_skip("timezone not available"); + return; + } + + int i; + const char* old_tz; + + QResults queries[] = {{"date:20080731..20080804", 5}, + {"date:2008-07-31..2008-08-04", 5}, + {"date:20080804..20080731", 5}, + {"date:20080731..20080804 s:gcc", 1}, + {"date:200808110803..now", 6}, + {"date:200808110803..today", 6}, + {"date:200808110801..now", 6}}; + old_tz = set_tz(la); + + TempDir tdir; + const auto xpath{make_database(tdir.path(), MU_TESTMAILDIR)}; + g_assert_false(xpath.empty()); + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + /* g_print ("%s\n", queries[i].query); */ + g_assert_cmpuint(run_and_count_matches(xpath, queries[i].query), + ==, queries[i].count); + } + + set_tz(old_tz); +} + +static void +test_mu_query_sizes(void) +{ + int i; + QResults queries[] = { + {"size:0b..2m", 19}, + {"size:3b..2m", 19}, + {"size:2k..4k", 4}, + + {"size:0b..2m", 19}, + {"size:2m..0b", 19}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_attach(void) +{ + int i; + QResults queries[] = {{"j:sittingbull.jpg", 1}, {"file:custer", 0}, {"file:custer.jpg", 1}}; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + if (g_test_verbose()) + mu_println("query: {}", queries[i].query); + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); + } +} + +static void +test_mu_query_msgid(void) +{ + int i; + QResults queries[] = { + {"i:CAHSaMxZ9rk5ASjqsbXizjTQuSk583=M6TORHz" + "=bfogtmbGGs5A@mail.gmail.com", + 1}, + {"msgid:CAHSaMxZ9rk5ASjqsbXizjTQuSk583=M6TORHz=" + "bfogtmbGGs5A@mail.gmail.com", + 1}, + + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + if (g_test_verbose()) + mu_println("query: {}", queries[i].query); + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); + } +} + +static void +test_mu_query_tags(void) +{ + int i; + QResults queries[] = { + {"x:paradise", 1}, + {"tag:lost", 1}, + {"tag:lost tag:paradise", 1}, + {"tag:lost tag:horizon", 0}, + {"tag:lost OR tag:horizon", 1}, + {"tag:queensryche", 1}, + {"tag:Queensrÿche", 1}, + {"x:paradise,lost", 0}, + {"x:paradise AND x:lost", 1}, + {"x:\\\\backslash", 1}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_wom_bat(void) +{ + int i; + QResults queries[] = { + {"maildir:/wom_bat", 3}, + //{ "\"maildir:/wom bat\"", 3}, + // as expected, no longer works with new parser + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_signed_encrypted(void) +{ + int i; + QResults queries[] = { + {"flag:encrypted", 2}, + {"flag:signed", 2}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, + queries[i].count); +} + +static void +test_mu_query_multi_to_cc(void) +{ + int i; + QResults queries[] = { + {"to:a@example.com", 1}, + {"cc:d@example.com", 1}, + {"to:b@example.com", 1}, + {"cc:e@example.com", 1}, + {"cc:e@example.com AND cc:d@example.com", 1}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_tags_02(void) +{ + int i; + QResults queries[] = { + {"x:paradise", 1}, + {"tag:@NextActions", 1}, + {"x:queensrÿche", 1}, + {"tag:lost OR tag:operation*", 2}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); + } +} + +/* Tests for https://github.com/djcb/mu/issues/380 + + On certain platforms, something goes wrong during compilation and the + --related option doesn't work. +*/ +static void +test_mu_query_threads_compilation_error(void) +{ + TempDir tdir; + const auto xpath = make_database(tdir.path(), MU_TESTMAILDIR); + + g_assert_cmpuint(run_and_count_matches(xpath, "msgid:uwsireh25.fsf@one.dot.net"), ==, 1); + + g_assert_cmpuint(run_and_count_matches(xpath, + "msgid:uwsireh25.fsf@one.dot.net", + QueryFlags::IncludeRelated), ==, 3); +} + +int +main(int argc, char* argv[]) +{ + TempDir td1; + TempDir td2; + + mu_test_init(&argc, &argv); + DB_PATH1 = make_database(td1.path(), MU_TESTMAILDIR); + g_assert_false(DB_PATH1.empty()); + + DB_PATH2 = make_database(td2.path(), MU_TESTMAILDIR2); + g_assert_false(DB_PATH2.empty()); + + g_test_add_func("/mu-query/test-mu-query-01", test_mu_query_01); + g_test_add_func("/mu-query/test-mu-query-02", test_mu_query_02); + g_test_add_func("/mu-query/test-mu-query-03", test_mu_query_03); + g_test_add_func("/mu-query/test-mu-query-04", test_mu_query_04); + + g_test_add_func("/mu-query/test-mu-query-signed-encrypted", test_mu_query_signed_encrypted); + g_test_add_func("/mu-query/test-mu-query-multi-to-cc", test_mu_query_multi_to_cc); + g_test_add_func("/mu-query/test-mu-query-logic", test_mu_query_logic); + + g_test_add_func("/mu-query/test-mu-query-accented-chars-1", + test_mu_query_accented_chars_01); + g_test_add_func("/mu-query/test-mu-query-accented-chars-2", + test_mu_query_accented_chars_02); + g_test_add_func("/mu-query/test-mu-query-accented-chars-fraiche", + test_mu_query_accented_chars_fraiche); + + g_test_add_func("/mu-query/test-mu-query-msgid", test_mu_query_msgid); + + g_test_add_func("/mu-query/test-mu-query-wom-bat", test_mu_query_wom_bat); + + g_test_add_func("/mu-query/test-mu-query-wildcards", test_mu_query_wildcards); + g_test_add_func("/mu-query/test-mu-query-sizes", test_mu_query_sizes); + + g_test_add_func("/mu-query/test-mu-query-dates-helsinki", test_mu_query_dates_helsinki); + g_test_add_func("/mu-query/test-mu-query-dates-sydney", test_mu_query_dates_sydney); + g_test_add_func("/mu-query/test-mu-query-dates-la", test_mu_query_dates_la); + + g_test_add_func("/mu-query/test-mu-query-attach", test_mu_query_attach); + g_test_add_func("/mu-query/test-mu-query-tags", test_mu_query_tags); + g_test_add_func("/mu-query/test-mu-query-tags_02", test_mu_query_tags_02); + + g_test_add_func("/mu-query/test-mu-query-threads-compilation-error", + test_mu_query_threads_compilation_error); + + return g_test_run(); +} diff --git a/mu4e/fdl.texi b/mu4e/fdl.texi new file mode 100644 index 0000000..96ce74e --- /dev/null +++ b/mu4e/fdl.texi @@ -0,0 +1,451 @@ +@c The GNU Free Documentation License. +@center Version 1.2, November 2002 + +@c This file is intended to be included within another document, +@c hence no sectioning command or @node. + +@display +Copyright @copyright{} 2000,2001,2002 Free Software Foundation, Inc. +51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. +@end display + +@enumerate 0 +@item +PREAMBLE + +The purpose of this License is to make a manual, textbook, or other +functional and useful document @dfn{free} in the sense of freedom: to +assure everyone the effective freedom to copy and redistribute it, +with or without modifying it, either commercially or noncommercially. +Secondarily, this License preserves for the author and publisher a way +to get credit for their work, while not being considered responsible +for modifications made by others. + +This License is a kind of ``copyleft'', which means that derivative +works of the document must themselves be free in the same sense. It +complements the GNU General Public License, which is a copyleft +license designed for free software. + +We have designed this License in order to use it for manuals for free +software, because free software needs free documentation: a free +program should come with manuals providing the same freedoms that the +software does. But this License is not limited to software manuals; +it can be used for any textual work, regardless of subject matter or +whether it is published as a printed book. We recommend this License +principally for works whose purpose is instruction or reference. + +@item +APPLICABILITY AND DEFINITIONS + +This License applies to any manual or other work, in any medium, that +contains a notice placed by the copyright holder saying it can be +distributed under the terms of this License. Such a notice grants a +world-wide, royalty-free license, unlimited in duration, to use that +work under the conditions stated herein. The ``Document'', below, +refers to any such manual or work. Any member of the public is a +licensee, and is addressed as ``you''. You accept the license if you +copy, modify or distribute the work in a way requiring permission +under copyright law. + +A ``Modified Version'' of the Document means any work containing the +Document or a portion of it, either copied verbatim, or with +modifications and/or translated into another language. + +A ``Secondary Section'' is a named appendix or a front-matter section +of the Document that deals exclusively with the relationship of the +publishers or authors of the Document to the Document's overall +subject (or to related matters) and contains nothing that could fall +directly within that overall subject. (Thus, if the Document is in +part a textbook of mathematics, a Secondary Section may not explain +any mathematics.) The relationship could be a matter of historical +connection with the subject or with related matters, or of legal, +commercial, philosophical, ethical or political position regarding +them. + +The ``Invariant Sections'' are certain Secondary Sections whose titles +are designated, as being those of Invariant Sections, in the notice +that says that the Document is released under this License. If a +section does not fit the above definition of Secondary then it is not +allowed to be designated as Invariant. The Document may contain zero +Invariant Sections. If the Document does not identify any Invariant +Sections then there are none. + +The ``Cover Texts'' are certain short passages of text that are listed, +as Front-Cover Texts or Back-Cover Texts, in the notice that says that +the Document is released under this License. A Front-Cover Text may +be at most 5 words, and a Back-Cover Text may be at most 25 words. + +A ``Transparent'' copy of the Document means a machine-readable copy, +represented in a format whose specification is available to the +general public, that is suitable for revising the document +straightforwardly with generic text editors or (for images composed of +pixels) generic paint programs or (for drawings) some widely available +drawing editor, and that is suitable for input to text formatters or +for automatic translation to a variety of formats suitable for input +to text formatters. A copy made in an otherwise Transparent file +format whose markup, or absence of markup, has been arranged to thwart +or discourage subsequent modification by readers is not Transparent. +An image format is not Transparent if used for any substantial amount +of text. A copy that is not ``Transparent'' is called ``Opaque''. + +Examples of suitable formats for Transparent copies include plain +@sc{ascii} without markup, Texinfo input format, La@TeX{} input +format, @acronym{SGML} or @acronym{XML} using a publicly available +@acronym{DTD}, and standard-conforming simple @acronym{HTML}, +PostScript or @acronym{PDF} designed for human modification. Examples +of transparent image formats include @acronym{PNG}, @acronym{XCF} and +@acronym{JPG}. Opaque formats include proprietary formats that can be +read and edited only by proprietary word processors, @acronym{SGML} or +@acronym{XML} for which the @acronym{DTD} and/or processing tools are +not generally available, and the machine-generated @acronym{HTML}, +PostScript or @acronym{PDF} produced by some word processors for +output purposes only. + +The ``Title Page'' means, for a printed book, the title page itself, +plus such following pages as are needed to hold, legibly, the material +this License requires to appear in the title page. For works in +formats which do not have any title page as such, ``Title Page'' means +the text near the most prominent appearance of the work's title, +preceding the beginning of the body of the text. + +A section ``Entitled XYZ'' means a named subunit of the Document whose +title either is precisely XYZ or contains XYZ in parentheses following +text that translates XYZ in another language. (Here XYZ stands for a +specific section name mentioned below, such as ``Acknowledgements'', +``Dedications'', ``Endorsements'', or ``History''.) To ``Preserve the Title'' +of such a section when you modify the Document means that it remains a +section ``Entitled XYZ'' according to this definition. + +The Document may include Warranty Disclaimers next to the notice which +states that this License applies to the Document. These Warranty +Disclaimers are considered to be included by reference in this +License, but only as regards disclaiming warranties: any other +implication that these Warranty Disclaimers may have is void and has +no effect on the meaning of this License. + +@item +VERBATIM COPYING + +You may copy and distribute the Document in any medium, either +commercially or noncommercially, provided that this License, the +copyright notices, and the license notice saying this License applies +to the Document are reproduced in all copies, and that you add no other +conditions whatsoever to those of this License. You may not use +technical measures to obstruct or control the reading or further +copying of the copies you make or distribute. However, you may accept +compensation in exchange for copies. If you distribute a large enough +number of copies you must also follow the conditions in section 3. + +You may also lend copies, under the same conditions stated above, and +you may publicly display copies. + +@item +COPYING IN QUANTITY + +If you publish printed copies (or copies in media that commonly have +printed covers) of the Document, numbering more than 100, and the +Document's license notice requires Cover Texts, you must enclose the +copies in covers that carry, clearly and legibly, all these Cover +Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on +the back cover. Both covers must also clearly and legibly identify +you as the publisher of these copies. The front cover must present +the full title with all words of the title equally prominent and +visible. You may add other material on the covers in addition. +Copying with changes limited to the covers, as long as they preserve +the title of the Document and satisfy these conditions, can be treated +as verbatim copying in other respects. + +If the required texts for either cover are too voluminous to fit +legibly, you should put the first ones listed (as many as fit +reasonably) on the actual cover, and continue the rest onto adjacent +pages. + +If you publish or distribute Opaque copies of the Document numbering +more than 100, you must either include a machine-readable Transparent +copy along with each Opaque copy, or state in or with each Opaque copy +a computer-network location from which the general network-using +public has access to download using public-standard network protocols +a complete Transparent copy of the Document, free of added material. +If you use the latter option, you must take reasonably prudent steps, +when you begin distribution of Opaque copies in quantity, to ensure +that this Transparent copy will remain thus accessible at the stated +location until at least one year after the last time you distribute an +Opaque copy (directly or through your agents or retailers) of that +edition to the public. + +It is requested, but not required, that you contact the authors of the +Document well before redistributing any large number of copies, to give +them a chance to provide you with an updated version of the Document. + +@item +MODIFICATIONS + +You may copy and distribute a Modified Version of the Document under +the conditions of sections 2 and 3 above, provided that you release +the Modified Version under precisely this License, with the Modified +Version filling the role of the Document, thus licensing distribution +and modification of the Modified Version to whoever possesses a copy +of it. In addition, you must do these things in the Modified Version: + +@enumerate A +@item +Use in the Title Page (and on the covers, if any) a title distinct +from that of the Document, and from those of previous versions +(which should, if there were any, be listed in the History section +of the Document). You may use the same title as a previous version +if the original publisher of that version gives permission. + +@item +List on the Title Page, as authors, one or more persons or entities +responsible for authorship of the modifications in the Modified +Version, together with at least five of the principal authors of the +Document (all of its principal authors, if it has fewer than five), +unless they release you from this requirement. + +@item +State on the Title page the name of the publisher of the +Modified Version, as the publisher. + +@item +Preserve all the copyright notices of the Document. + +@item +Add an appropriate copyright notice for your modifications +adjacent to the other copyright notices. + +@item +Include, immediately after the copyright notices, a license notice +giving the public permission to use the Modified Version under the +terms of this License, in the form shown in the Addendum below. + +@item +Preserve in that license notice the full lists of Invariant Sections +and required Cover Texts given in the Document's license notice. + +@item +Include an unaltered copy of this License. + +@item +Preserve the section Entitled ``History'', Preserve its Title, and add +to it an item stating at least the title, year, new authors, and +publisher of the Modified Version as given on the Title Page. If +there is no section Entitled ``History'' in the Document, create one +stating the title, year, authors, and publisher of the Document as +given on its Title Page, then add an item describing the Modified +Version as stated in the previous sentence. + +@item +Preserve the network location, if any, given in the Document for +public access to a Transparent copy of the Document, and likewise +the network locations given in the Document for previous versions +it was based on. These may be placed in the ``History'' section. +You may omit a network location for a work that was published at +least four years before the Document itself, or if the original +publisher of the version it refers to gives permission. + +@item +For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve +the Title of the section, and preserve in the section all the +substance and tone of each of the contributor acknowledgements and/or +dedications given therein. + +@item +Preserve all the Invariant Sections of the Document, +unaltered in their text and in their titles. Section numbers +or the equivalent are not considered part of the section titles. + +@item +Delete any section Entitled ``Endorsements''. Such a section +may not be included in the Modified Version. + +@item +Do not retitle any existing section to be Entitled ``Endorsements'' or +to conflict in title with any Invariant Section. + +@item +Preserve any Warranty Disclaimers. +@end enumerate + +If the Modified Version includes new front-matter sections or +appendices that qualify as Secondary Sections and contain no material +copied from the Document, you may at your option designate some or all +of these sections as invariant. To do this, add their titles to the +list of Invariant Sections in the Modified Version's license notice. +These titles must be distinct from any other section titles. + +You may add a section Entitled ``Endorsements'', provided it contains +nothing but endorsements of your Modified Version by various +parties---for example, statements of peer review or that the text has +been approved by an organization as the authoritative definition of a +standard. + +You may add a passage of up to five words as a Front-Cover Text, and a +passage of up to 25 words as a Back-Cover Text, to the end of the list +of Cover Texts in the Modified Version. Only one passage of +Front-Cover Text and one of Back-Cover Text may be added by (or +through arrangements made by) any one entity. If the Document already +includes a cover text for the same cover, previously added by you or +by arrangement made by the same entity you are acting on behalf of, +you may not add another; but you may replace the old one, on explicit +permission from the previous publisher that added the old one. + +The author(s) and publisher(s) of the Document do not by this License +give permission to use their names for publicity for or to assert or +imply endorsement of any Modified Version. + +@item +COMBINING DOCUMENTS + +You may combine the Document with other documents released under this +License, under the terms defined in section 4 above for modified +versions, provided that you include in the combination all of the +Invariant Sections of all of the original documents, unmodified, and +list them all as Invariant Sections of your combined work in its +license notice, and that you preserve all their Warranty Disclaimers. + +The combined work need only contain one copy of this License, and +multiple identical Invariant Sections may be replaced with a single +copy. If there are multiple Invariant Sections with the same name but +different contents, make the title of each such section unique by +adding at the end of it, in parentheses, the name of the original +author or publisher of that section if known, or else a unique number. +Make the same adjustment to the section titles in the list of +Invariant Sections in the license notice of the combined work. + +In the combination, you must combine any sections Entitled ``History'' +in the various original documents, forming one section Entitled +``History''; likewise combine any sections Entitled ``Acknowledgements'', +and any sections Entitled ``Dedications''. You must delete all +sections Entitled ``Endorsements.'' + +@item +COLLECTIONS OF DOCUMENTS + +You may make a collection consisting of the Document and other documents +released under this License, and replace the individual copies of this +License in the various documents with a single copy that is included in +the collection, provided that you follow the rules of this License for +verbatim copying of each of the documents in all other respects. + +You may extract a single document from such a collection, and distribute +it individually under this License, provided you insert a copy of this +License into the extracted document, and follow this License in all +other respects regarding verbatim copying of that document. + +@item +AGGREGATION WITH INDEPENDENT WORKS + +A compilation of the Document or its derivatives with other separate +and independent documents or works, in or on a volume of a storage or +distribution medium, is called an ``aggregate'' if the copyright +resulting from the compilation is not used to limit the legal rights +of the compilation's users beyond what the individual works permit. +When the Document is included in an aggregate, this License does not +apply to the other works in the aggregate which are not themselves +derivative works of the Document. + +If the Cover Text requirement of section 3 is applicable to these +copies of the Document, then if the Document is less than one half of +the entire aggregate, the Document's Cover Texts may be placed on +covers that bracket the Document within the aggregate, or the +electronic equivalent of covers if the Document is in electronic form. +Otherwise they must appear on printed covers that bracket the whole +aggregate. + +@item +TRANSLATION + +Translation is considered a kind of modification, so you may +distribute translations of the Document under the terms of section 4. +Replacing Invariant Sections with translations requires special +permission from their copyright holders, but you may include +translations of some or all Invariant Sections in addition to the +original versions of these Invariant Sections. You may include a +translation of this License, and all the license notices in the +Document, and any Warranty Disclaimers, provided that you also include +the original English version of this License and the original versions +of those notices and disclaimers. In case of a disagreement between +the translation and the original version of this License or a notice +or disclaimer, the original version will prevail. + +If a section in the Document is Entitled ``Acknowledgements'', +``Dedications'', or ``History'', the requirement (section 4) to Preserve +its Title (section 1) will typically require changing the actual +title. + +@item +TERMINATION + +You may not copy, modify, sublicense, or distribute the Document except +as expressly provided for under this License. Any other attempt to +copy, modify, sublicense or distribute the Document is void, and will +automatically terminate your rights under this License. However, +parties who have received copies, or rights, from you under this +License will not have their licenses terminated so long as such +parties remain in full compliance. + +@item +FUTURE REVISIONS OF THIS LICENSE + +The Free Software Foundation may publish new, revised versions +of the GNU Free Documentation License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. See +@uref{http://www.gnu.org/copyleft/}. + +Each version of the License is given a distinguishing version number. +If the Document specifies that a particular numbered version of this +License ``or any later version'' applies to it, you have the option of +following the terms and conditions either of that specified version or +of any later version that has been published (not as a draft) by the +Free Software Foundation. If the Document does not specify a version +number of this License, you may choose any version ever published (not +as a draft) by the Free Software Foundation. +@end enumerate + +@page +@heading ADDENDUM: How to use this License for your documents + +To use this License in a document you have written, include a copy of +the License in the document and put the following copyright and +license notices just after the title page: + +@smallexample +@group + Copyright (C) @var{year} @var{your name}. + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU Free Documentation License, Version 1.2 + or any later version published by the Free Software Foundation; + with no Invariant Sections, no Front-Cover Texts, and no Back-Cover + Texts. A copy of the license is included in the section entitled ``GNU + Free Documentation License''. +@end group +@end smallexample + +If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts, +replace the ``with@dots{}Texts.'' line with this: + +@smallexample +@group + with the Invariant Sections being @var{list their titles}, with + the Front-Cover Texts being @var{list}, and with the Back-Cover Texts + being @var{list}. +@end group +@end smallexample + +If you have Invariant Sections without Cover Texts, or some other +combination of the three, merge those two alternatives to suit the +situation. + +If your document contains nontrivial examples of program code, we +recommend releasing these examples in parallel under your choice of +free software license, such as the GNU General Public License, +to permit their use in free software. + +@c Local Variables: +@c ispell-local-pdict: "ispell-dict" +@c End: + diff --git a/mu4e/htmlxref.cnf b/mu4e/htmlxref.cnf new file mode 100644 index 0000000..1af587b --- /dev/null +++ b/mu4e/htmlxref.cnf @@ -0,0 +1,788 @@ +# htmlxref.cnf - reference file for free Texinfo manuals on the web. + +htmlxrefversion=2023-04-02.12; # UTC + +# Copyright 2010-2023 Free Software Foundation, Inc. +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# +# The latest version of this file is available at +# http://ftpmirror.gnu.org/texinfo/htmlxref.cnf. +# Email corrections or additions to bug-texinfo@gnu.org. +# The primary goal is to list all relevant GNU manuals; +# other free manuals are also welcome. +# +# To be included in this list, a manual must: +# +# - have a generic url, e.g., no version numbers; +# - have a unique file name (e.g., manual identifier), i.e., be related to the +# package name. Things like "refman" or "tutorial" don't work. +# - follow the naming convention for nodes described at +# http://www.gnu.org/software/texinfo/manual/texinfo/html_node/HTML-Xref.html +# This is what makeinfo and texi2html implement. +# +# Unless the above criteria are met, it's not possible to generate +# reliable cross-manual references. +# +# For information on automatically generating all the useful formats for +# a manual to put on the web, see +# http://www.gnu.org/prep/maintain/html_node/Manuals-on-Web-Pages.html. + +# For people editing this file: when a manual named foo is related to a +# package named bar, the url should contain a variable reference ${BAR}. +# Otherwise, the gnumaint scripts have no way of knowing they are +# associated, and thus gnu.org/manual can't include them. + +# shorten references to manuals on www.gnu.org. +G = https://www.gnu.org +GS = ${G}/software + +3dldf mono ${GS}/3dldf/manual/user_ref/3DLDF.html +3dldf node ${GS}/3dldf/manual/user_ref/ + +alive mono ${GS}/alive/manual/alive.html +alive node ${GS}/alive/manual/html_node/ + +anubis mono ${GS}/anubis/manual/anubis.html +anubis chapter ${GS}/anubis/manual/html_chapter/ +anubis section ${GS}/anubis/manual/html_section/ +anubis node ${GS}/anubis/manual/html_node/ + +artanis mono ${GS}/artanis/manual/artanis.html +artanis node ${GS}/artanis/manual/html_node/ + +aspell section http://aspell.net/man-html/index.html + +auctex mono ${GS}/auctex/manual/auctex.html +auctex node ${GS}/auctex/manual/auctex/ + +autoconf mono ${GS}/autoconf/manual/autoconf.html +autoconf node ${GS}/autoconf/manual/html_node/ + +autogen mono ${GS}/autogen/manual/autogen.html +autogen chapter ${GS}/autogen/manual/html_chapter/ +autogen node ${GS}/autoconf/manual/html_node/ + +automake mono ${GS}/automake/manual/automake.html +automake node ${GS}/automake/manual/html_node/ + +avl node http://adtinfo.org/libavl.html/ + +bash mono ${GS}/bash/manual/bash.html +bash node ${GS}/bash/manual/html_node/ + +BINUTILS = https://sourceware.org/binutils/docs +binutils mono ${BINUTILS}/binutils.html +binutils node ${BINUTILS}/binutils/ + # + as mono ${BINUTILS}/as.html + as node ${BINUTILS}/as/ + # + bfd mono ${BINUTILS}/bfd.html + bfd node ${BINUTILS}/bfd/ + # + gprof mono ${BINUTILS}/gprof.html + gprof node ${BINUTILS}/gprof/ + # + ld mono ${BINUTILS}/ld.html + ld node ${BINUTILS}/ld/ + +bison mono ${GS}/bison/manual/bison.html +bison node ${GS}/bison/manual/html_node/ + +bpel2owfn mono ${GS}/bpel2owfn/manual/2.0.x/bpel2owfn.html + +ccd2cue mono ${GS}/ccd2cue/manual/ccd2cue.html +ccd2cue node ${GS}/ccd2cue/manual/html_node/ + +cflow mono ${GS}/cflow/manual/cflow.html +cflow node ${GS}/cflow/manual/html_node/ + +chess mono ${GS}/chess/manual/gnuchess.html +chess node ${GS}/chess/manual/html_node/ + +combine mono ${GS}/combine/manual/combine.html +combine chapter ${GS}/combine/manual/html_chapter/ +combine section ${GS}/combine/manual/html_section/ +combine node ${GS}/combine/manual/html_node/ + +complexity mono ${GS}/complexity/manual/complexity.html +complexity node ${GS}/complexity/manual/html_node/ + +coreutils mono ${GS}/coreutils/manual/coreutils.html +coreutils node ${GS}/coreutils/manual/html_node/ + +cpio mono ${GS}/cpio/manual/cpio.html +cpio node ${GS}/cpio/manual/html_node/ + +cssc node ${GS}/cssc/manual/ + +CVS = ${GS}/trans-coord/manual +cvs mono ${CVS}/cvs/cvs.html +cvs node ${CVS}/cvs/html_node/ + +ddd mono ${GS}/ddd/manual/html_mono/ddd.html + +ddrescue mono ${GS}/ddrescue/manual/ddrescue_manual.html + +dejagnu node ${GS}/dejagnu/manual/ + +DICO = https://www.gnu.org.ua/software/dico/manual +dico mono ${DICO}/dico.html +dico chapter ${DICO}/html_chapter/ +dico section ${DICO}/html_section/ +dico node ${DICO}/html_node/ + +diffutils mono ${GS}/diffutils/manual/diffutils.html +diffutils node ${GS}/diffutils/manual/html_node/ + +ed mono ${GS}/ed/manual/ed_manual.html + +EMACS = ${GS}/emacs/manual +emacs mono ${EMACS}/html_mono/emacs.html +emacs node ${EMACS}/html_node/emacs/ + # + auth mono ${EMACS}/html_mono/auth.html + auth node ${EMACS}/html_node/auth/ + # + autotype mono ${EMACS}/html_mono/autotype.html + autotype node ${EMACS}/html_node/autotype/ + # + calc mono ${EMACS}/html_mono/calc.html + calc node ${EMACS}/html_node/calc/ + # + ccmode mono ${EMACS}/html_mono/ccmode.html + ccmode node ${EMACS}/html_node/ccmode/ + # + cl mono ${EMACS}/html_mono/cl.html + cl node ${EMACS}/html_node/cl/ + # + dbus mono ${EMACS}/html_mono/dbus.html + dbus node ${EMACS}/html_node/dbus/ + # + ebrowse mono ${EMACS}/html_mono/ebrowse.html + ebrowse node ${EMACS}/html_node/ebrowse/ + # + ede mono ${EMACS}/html_mono/ede.html + ede node ${EMACS}/html_node/ede/ + # + edt mono ${EMACS}/html_mono/edt.html + edt node ${EMACS}/html_node/edt/ + # + ediff mono ${EMACS}/html_mono/ediff.html + ediff node ${EMACS}/html_node/ediff/ + # + eieio mono ${EMACS}/html_mono/eieio.html + eieio node ${EMACS}/html_node/eieio/ + # + elisp mono ${EMACS}/html_mono/elisp.html + elisp node ${EMACS}/html_node/elisp/ + # + emacs-gnutls mono ${EMACS}/html_mono/emacs-gnutls.html + emacs-gnutls node ${EMACS}/html_node/emacs-gnutls/ + # + emacs-mime mono ${EMACS}/html_mono/emacs-mime.html + emacs-mime node ${EMACS}/html_node/emacs-mime/ + # + epa mono ${EMACS}/html_mono/epa.html + epa node ${EMACS}/html_node/epa/ + # + erc mono ${EMACS}/html_mono/erc.html + erc node ${EMACS}/html_node/erc/ + # + dired-x mono ${EMACS}/html_mono/dired-x.html + dired-x node ${EMACS}/html_node/dired-x/ + # + ert mono ${EMACS}/html_mono/ert.html + ert node ${EMACS}/html_node/ert/ + # + eshell mono ${EMACS}/html_mono/eshell.html + eshell node ${EMACS}/html_node/eshell/ + # + eudc mono ${EMACS}/html_mono/eudc.html + eudc node ${EMACS}/html_node/eudc/ + # + eww mono ${EMACS}/html_mono/eww.html + eww node ${EMACS}/html_node/eww/ + # + forms mono ${EMACS}/html_mono/forms.html + forms node ${EMACS}/html_node/forms/ + # + flymake mono ${EMACS}/html_mono/flymake.html + flymake node ${EMACS}/html_node/flymake/ + # + gnus mono ${EMACS}/html_mono/gnus.html + gnus node ${EMACS}/html_node/gnus/ + # + htmlfontify mono ${EMACS}/html_mono/htmlfontify.html + htmlfontify node ${EMACS}/html_node/htmlfontify/ + # + idlwave mono ${EMACS}/html_mono/idlwave.html + idlwave node ${EMACS}/html_node/idlwave/ + # + ido mono ${EMACS}/html_mono/ido.html + ido node ${EMACS}/html_node/ido/ + # + info mono ${EMACS}/html_mono/info.html + info node ${EMACS}/html_node/info/ + # + mairix-el mono ${EMACS}/html_mono/mairix-el.html + mairix-el node ${EMACS}/html_node/mairix-el/ + # + message mono ${EMACS}/html_mono/message.html + message node ${EMACS}/html_node/message/ + # + mh-e mono ${EMACS}/html_mono/mh-e.html + mh-e node ${EMACS}/html_node/mh-e/ + # + newsticker mono ${EMACS}/html_mono/newsticker.html + newsticker node ${EMACS}/html_node/newsticker/ + # + nxml-mode mono ${EMACS}/html_mono/nxml-mode.html + nxml-mode node ${EMACS}/html_node/nxml-mode/ + # + octave-mode mono ${EMACS}/html_mono/octave-mode.html + octave-mode node ${EMACS}/html_node/octave-mode/ + # + org mono ${EMACS}/html_mono/org.html + org node ${EMACS}/html_node/org/ + # + pcl-cvs mono ${EMACS}/html_mono/pcl-cvs.html + pcl-cvs node ${EMACS}/html_node/pcl-cvs/ + # + pgg mono ${EMACS}/html_mono/pgg.html + pgg node ${EMACS}/html_node/pgg/ + # + rcirc mono ${EMACS}/html_mono/rcirc.html + rcirc node ${EMACS}/html_node/rcirc/ + # + reftex mono ${EMACS}/html_mono/reftex.html + reftex node ${EMACS}/html_node/reftex/ + # + remember mono ${EMACS}/html_mono/remember.html + remember node ${EMACS}/html_node/remember/ + # + sasl mono ${EMACS}/html_mono/sasl.html + sasl node ${EMACS}/html_node/sasl/ + # + semantic mono ${EMACS}/html_mono/semantic.html + semantic node ${EMACS}/html_node/semantic/ + # + bovine mono ${EMACS}/html_mono/bovine.html + bovine node ${EMACS}/html_node/bovine/ + # + srecode mono ${EMACS}/html_mono/srecode.html + srecode node ${EMACS}/html_node/srecode/ + # + ses mono ${EMACS}/html_mono/ses.html + ses node ${EMACS}/html_node/ses/ + # + sieve mono ${EMACS}/html_mono/sieve.html + sieve node ${EMACS}/html_node/sieve/ + # + smtp mono ${EMACS}/html_mono/smtpmail.html + smtp node ${EMACS}/html_node/smtpmail/ + # + speedbar mono ${EMACS}/html_mono/speedbar.html + speedbar node ${EMACS}/html_node/speedbar/ + # + sc mono ${EMACS}/html_mono/sc.html + sc node ${EMACS}/html_node/sc/ + # + todo-mode mono ${EMACS}/html_mono/todo-mode.html + todo-mode node ${EMACS}/html_node/todo-mode/ + # + tramp mono ${EMACS}/html_mono/tramp.html + tramp node ${EMACS}/html_node/tramp/ + # + url mono ${EMACS}/html_mono/url.html + url node ${EMACS}/html_node/url/ + # + vhdl-mode mono ${EMACS}/html_mono/vhdl-mode.html + vhdl-mode node ${EMACS}/html_node/vhdl-mode/ + # + vip mono ${EMACS}/html_mono/vip.html + vip node ${EMACS}/html_node/vip/ + # + viper mono ${EMACS}/html_mono/viper.html + viper node ${EMACS}/html_node/viper/ + # + widget mono ${EMACS}/html_mono/widget.html + widget node ${EMACS}/html_node/widget/ + # + wisent mono ${EMACS}/html_mono/wisent.html + wisent node ${EMACS}/html_node/wisent/ + # + woman mono ${EMACS}/html_mono/woman.html + woman node ${EMACS}/html_node/woman/ + # (end emacs manuals in EMACS) + +easejs mono ${GS}/easejs/manual/easejs.html +easejs node ${GS}/easejs/manual/ + +emacs-muse mono ${GS}/emacs-muse/manual/muse.html +emacs-muse node ${GS}/emacs-muse/manual/html_node/ + +emms node ${GS}/emms/manual/ + +ada-mode mono https://elpa.gnu.org/packages/ada-mode.html + +gpr-mode mono https://elpa.gnu.org/packages/doc/gpr-mode.html + +findutils mono ${GS}/findutils/manual/html_mono/find.html +findutils node ${GS}/findutils/manual/html_node/find_html + +flex node https://westes.github.io/flex/manual/ + +gama mono ${GS}/gama/manual/gama.html +gama node ${GS}/gama/manual/html_node/ + +GAWK = ${GS}/gawk/manual +gawk mono ${GAWK}/gawk.html +gawk node ${GAWK}/html_node/ + gawkinet mono ${GAWK}/gawkinet/gawkinet.html + gawkinet node ${GAWK}/gawkinet/html_node/ + +gcal mono ${GS}/gcal/manual/gcal.html +gcal node ${GS}/gcal/manual/html_node/ + +GCC = https://gcc.gnu.org/onlinedocs +gcc node ${GCC}/gcc/ + cpp node ${GCC}/cpp/ + gfortran node ${GCC}/gfortran/ + gnat_rm node ${GCC}/gnat_rm/ + gnat_ugn node ${GCC}/gnat_ugn/ + libgomp node ${GCC}/libgomp/ + libstdc++ node ${GCC}/libstdc++/ + # + gccint node ${GCC}/gccint/ + cppinternals node ${GCC}/cppinternals/ + gfc-internals node ${GCC}/gfc-internals/ + gnat-style node ${GCC}/gnat-style/ + libiberty node ${GCC}/libiberty/ + +GDB = https://sourceware.org/gdb/current/onlinedocs +gdb node ${GDB}/gdb.html/ + stabs node ${GDB}/stabs.html/ + +GDBM = http://www.gnu.org.ua/software/gdbm/manual +gdbm node ${GDBM}/ + +gettext mono ${GS}/gettext/manual/gettext.html +gettext node ${GS}/gettext/manual/html_node/ + +gforth node https://www.complang.tuwien.ac.at/forth/gforth/Docs-html/ + +global mono ${GS}/global/manual/global.html + +gmediaserver node ${GS}/gmediaserver/manual/ + +gmp node https://www.gmplib.org/manual/ + +gnu-arch node ${GS}/gnu-arch/tutorial/ + +gnu-c-manual mono ${GS}/gnu-c-manual/gnu-c-manual.html + +gnu-crypto node ${GS}/gnu-crypto/manual/ + +gnubg mono ${GS}/gnubg/manual/gnubg.html +gnubg node ${GS}/gnubg/manual/html_node/ + +GNUCOBOL = https://gnucobol.sourceforge.io/HTML +gnucobpg mono ${GNUCOBOL}/gnucobpg.html + gnucobqr mono ${GNUCOBOL}/gnucobqr.html + gnucobsp mono ${GNUCOBOL}/gnucobsp.html + +gnubik mono ${GS}/gnubik/manual/gnubik.html +gnubik node ${GS}/gnubik/manual/html_node/ + +gnulib mono ${GS}/gnulib/manual/gnulib.html +gnulib node ${GS}/gnulib/manual/html_node/ + +GNUN = ${GS}/trans-coord/manual +gnun mono ${GNUN}/gnun/gnun.html +gnun node ${GNUN}/gnun/html_node/ + web-trans mono ${GNUN}/web-trans/web-trans.html + web-trans node ${GNUN}/web-trans/html_node/ + +GNUPG = https://www.gnupg.org/documentation/manuals +gnupg node ${GNUPG}/gnupg/ + dirmngr node ${GNUPG}/dirmngr/ + gcrypt node ${GNUPG}/gcrypt/ + libgcrypt node ${GNUPG}/gcrypt/ + ksba node ${GNUPG}/ksba/ + assuan node ${GNUPG}/assuan/ + gpgme node ${GNUPG}/gpgme/ + +gnuprologjava node ${GS}/gnuprologjava/manual/ + +gnuschool mono ${GS}/gnuschool/gnuschool.html + +GNUSTANDARDS = ${G}/prep + maintain mono ${GNUSTANDARDS}/maintain/maintain.html + maintain node ${GNUSTANDARDS}/maintain/html_node/ + # + standards mono ${GNUSTANDARDS}/standards/standards.html + standards node ${GNUSTANDARDS}/standards/html_node/ + +# following url is a redirect, which cannot be used for links within the manual +#gnutls mono ${GS}/gnutls/manual/gnutls.html +# empty directory +#gnutls node ${GS}/gnutls/manual/html_node/ +GNUTLS = http://www.gnutls.org/manual +gnutls mono ${GNUTLS}/gnutls.html +gnutls node ${GNUTLS}/html_node/ + +gperf mono ${GS}/gperf/manual/gperf.html +gperf node ${GS}/gperf/manual/html_node/ + +grep mono ${GS}/grep/manual/grep.html +grep node ${GS}/grep/manual/html_node/ + +groff node ${GS}/groff/manual/html_node/ + +GRUB = ${GS}/grub/manual/ + grub mono ${GRUB}/grub/grub.html + grub node ${GRUB}/grub/html_node/ + # + multiboot mono ${GRUB}/multiboot/multiboot.html + multiboot node ${GRUB}/multiboot/html_node/ + # + grub-dev mono ${GRUB}/grub-dev/grub-dev.html + grub-dev node ${GRUB}/grub-dev/html_node/ + +gsasl mono ${GS}/gsasl/manual/gsasl.html +gsasl node ${GS}/gsasl/manual/html_node/ + +gsl node ${GS}/gsl/manual/html_node/ + +gsrc mono ${GS}/gsrc/manual/gsrc.html +gsrc node ${GS}/gsrc/manual/html_node/ + +gss mono ${GS}/gss/manual/gss.html +gss node ${GS}/gss/manual/html_node/ + +gtypist mono ${GS}/gtypist/doc/gtypist.html + +guile mono ${GS}/guile/manual/guile.html +guile node ${GS}/guile/manual/html_node/ + +GUILE_GNOME = ${GS}/guile-gnome/docs + gobject node ${GUILE_GNOME}/gobject/html/ + glib node ${GUILE_GNOME}/glib/html/ + atk node ${GUILE_GNOME}/atk/html/ + pango node ${GUILE_GNOME}/pango/html/ + pangocairo node ${GUILE_GNOME}/pangocairo/html/ + gdk node ${GUILE_GNOME}/gdk/html/ + gtk node ${GUILE_GNOME}/gtk/html/ + libglade node ${GUILE_GNOME}/libglade/html/ + gnome-vfs node ${GUILE_GNOME}/gnome-vfs/html/ + libgnomecanvas node ${GUILE_GNOME}/libgnomecanvas/html/ + gconf node ${GUILE_GNOME}/gconf/html/ + libgnome node ${GUILE_GNOME}/libgnome/html/ + libgnomeui node ${GUILE_GNOME}/libgnomeui/html/ + corba node ${GUILE_GNOME}/corba/html/ + clutter node ${GUILE_GNOME}/clutter/html/ + clutter-glx node ${GUILE_GNOME}/clutter-glx/html/ + +guile-gtk node ${GS}/guile-gtk/docs/guile-gtk/ + +guile-rpc mono ${GS}/guile-rpc/manual/guile-rpc.html +guile-rpc node ${GS}/guile-rpc/manual/html_node/ + +guix mono ${GS}/guix/manual/guix.html +guix node ${GS}/guix/manual/html_node/ + +gv mono ${GS}/gv/manual/gv.html +gv node ${GS}/gv/manual/html_node/ + +gzip mono ${GS}/gzip/manual/gzip.html +gzip node ${GS}/gzip/manual/html_node/ + +hello mono ${GS}/hello/manual/hello.html +hello node ${GS}/hello/manual/html_node/ + +help2man mono ${GS}/help2man/help2man.html + +idutils mono ${GS}/idutils/manual/idutils.html +idutils node ${GS}/idutils/manual/html_node/ + +inetutils mono ${GS}/inetutils/manual/inetutils.html +inetutils node ${GS}/inetutils/manual/html_node/ + +# No manual, redirects to git sources +#jwhois mono ${GS}/jwhois/manual/jwhois.html +# 404 Not Found +#jwhois node ${GS}/jwhois/manual/html_node/ + +libc mono ${GS}/libc/manual/html_mono/libc.html +libc node ${GS}/libc/manual/html_node/ + +LIBCDIO = ${GS}/libcdio + libcdio mono ${LIBCDIO}/libcdio.html + cd-text mono ${LIBCDIO}/cd-text-format.html + +libextractor mono ${GS}/libextractor/manual/libextractor.html +libextractor node ${GS}/libextractor/manual/html_node/ + +libidn mono ${GS}/libidn/manual/libidn.html +libidn node ${GS}/libidn/manual/html_node/ + +libidn2 mono ${GS}/libidn/libidn2/manual/libidn2.html +libidn2 node ${GS}/libidn/libidn2/manual/html_node/ + +librejs mono ${GS}/librejs/manual/librejs.html +librejs node ${GS}/librejs/manual/html_node/ + +libmatheval mono ${GS}/libmatheval/manual/libmatheval.html + +LIBMICROHTTPD = ${GS}/libmicrohttpd +libmicrohttpd mono ${LIBMICROHTTPD}/manual/libmicrohttpd.html +libmicrohttpd node ${LIBMICROHTTPD}/manual/html_node/ + # The manual name is based on the Texinfo file name in the code, + # not on the file name for the tutorial which is too generic. + microhttpd-tutorial mono ${LIBMICROHTTPD}/tutorial.html + +libtasn1 mono ${GS}/libtasn1/manual/libtasn1.html +libtasn1 node ${GS}/libtasn1/manual/html_node/ + +libtool mono ${GS}/libtool/manual/libtool.html +libtool node ${GS}/libtool/manual/html_node/ + +lightning mono ${GS}/lightning/manual/lightning.html +lightning node ${GS}/lightning/manual/html_node/ + +# The stable/ url redirects immediately, but that's ok. +# The .html extension is omitted on their web site, but it works if given. +LILYPOND = http://lilypond.org/doc/stable/Documentation + lilypond-internals node ${LILYPOND}/internals/ + lilypond-learning node ${LILYPOND}/learning/ + lilypond-notation node ${LILYPOND}/notation/ + lilypond-snippets node ${LILYPOND}/snippets/ + lilypond-usage node ${LILYPOND}/usage/ + lilypond-web node ${LILYPOND}/web/ + music-glossary node ${LILYPOND}/music-glossary/ + +liquidwar6 mono ${GS}/liquidwar6/manual/liquidwar6.html +liquidwar6 node ${GS}/liquidwar6/manual/html_node/ + +lispintro mono ${GS}/emacs/emacs-lisp-intro/html_mono/emacs-lisp-intro.html +lispintro node ${GS}/emacs/emacs-lisp-intro/html_node/index.html + +LSH = http://www.lysator.liu.se/~nisse/lsh + lsh mono ${LSH}/lsh.html + +m4 mono ${GS}/m4/manual/m4.html +m4 node ${GS}/m4/manual/html_node/ + +MITGNUSCHEME = ${GS}/mit-scheme/documentation/stable +mit-scheme-user mono ${MITGNUSCHEME}/mit-scheme-user.html +mit-scheme-user node ${MITGNUSCHEME}/mit-scheme-user/ + # + mit-scheme-ref mono ${MITGNUSCHEME}/mit-scheme-ref.html + mit-scheme-ref node ${MITGNUSCHEME}/mit-scheme-ref/ + # + mit-scheme-ffi mono ${MITGNUSCHEME}/mit-scheme-ffi.html + mit-scheme-ffi node ${MITGNUSCHEME}/mit-scheme-ffi/ + # + mit-scheme-sos mono ${MITGNUSCHEME}/mit-scheme-sos.html + mit-scheme-sos node ${MITGNUSCHEME}/mit-scheme-sos/ + # + mit-scheme-imail mono ${MITGNUSCHEME}/mit-scheme-imail.html + # + mit-scheme-blowfish mono ${MITGNUSCHEME}/mit-scheme-blowfish.html + # + mit-scheme-gdbm mono ${MITGNUSCHEME}/mit-scheme-gdbm.html + +mailutils mono ${GS}/mailutils/manual/mailutils.html +mailutils chapter ${GS}/mailutils/manual/html_chapter/ +mailutils section ${GS}/mailutils/manual/html_section/ +mailutils node ${GS}/mailutils/manual/html_node/ + +make mono ${GS}/make/manual/make.html +make node ${GS}/make/manual/html_node/ + +mdk mono ${GS}/mdk/manual/mdk.html +mdk node ${GS}/mdk/manual/html_node/ + +METAEXCHANGE = https://ftp.gwdg.de/pub/gnu2/iwfmdh/doc/texinfo + iwf_mh node ${METAEXCHANGE}/iwf_mh.html + scantest node ${METAEXCHANGE}/scantest.html + +MIT_SCHEME = ${GS}/mit-scheme/documentation/stable + mit-scheme-ref node ${MIT_SCHEME}/mit-scheme-ref/ + mit-scheme-user node ${MIT_SCHEME}/mit-scheme-user/ + sos node ${MIT_SCHEME}/mit-scheme-sos/ + mit-scheme-imail mono ${MIT_SCHEME}/mit-scheme-imail.html + +moe mono ${GS}/moe/manual/moe_manual.html + +motti node ${GS}/motti/manual/ + +# only PDF is available as documentation to download +#mpc node http://www.multiprecision.org/index.php?prog=mpc&page=html + +mpfr mono https://www.mpfr.org/mpfr-current/mpfr.html + +mtools mono ${GS}/mtools/manual/mtools.html + +nano mono https://www.nano-editor.org/dist/latest/nano.html + +nettle mono https://www.lysator.liu.se/~nisse/nettle/nettle.html + +ocrad mono ${GS}/ocrad/manual/ocrad_manual.html + +parted mono ${GS}/parted/manual/parted.html +parted node ${GS}/parted/manual/html_node/ + +pascal node https://www.gnu-pascal.de/gpc/ + +# can't use pcb since url's contain dates --30nov10 + +PIES = http://www.gnu.org.ua/software/pies/manual +pies node ${PIES}/ + +plotutils mono ${GS}/plotutils/manual/en/plotutils.html +plotutils node ${GS}/plotutils/manual/en/html_node/ + +proxyknife mono ${GS}/proxyknife/manual/proxyknife.html +proxyknife node ${GS}/proxyknife/manual/html_node/ + +pspp mono ${GS}/pspp/manual/pspp.html +pspp node ${GS}/pspp/manual/html_node/ + +pyconfigure mono ${GS}/pyconfigure/manual/pyconfigure.html +pyconfigure node ${GS}/pyconfigure/manual/html_node/ + +R = https://cran.r-project.org/doc/manuals + R-intro mono ${R}/R-intro.html + R-lang mono ${R}/R-lang.html + R-exts mono ${R}/R-exts.html + R-data mono ${R}/R-data.html + R-admin mono ${R}/R-admin.html + R-ints mono ${R}/R-ints.html + +rcs mono ${GS}/rcs/manual/rcs.html +rcs node ${GS}/rcs/manual/html_node/ + +READLINE = https://tiswww.cwru.edu/php/chet/readline +readline mono ${READLINE}/readline.html + rluserman mono ${READLINE}/rluserman.html + history mono ${READLINE}/history.html + +# no manual for Recode found. Most recent fork seems to be at +# https://github.com/rrthomas/recode/ + +recutils mono ${GS}/recutils/manual/recutils.html +recutils node ${GS}/recutils/manual/html_node/ + +remotecontrol mono ${GS}/remotecontrol/manual/remotecontrol.html +remotecontrol node ${GS}/remotecontrol/manual/html_node/ + +rottlog mono ${GS}/rottlog/manual/rottlog.html +rottlog node ${GS}/rottlog/manual/html_node/ + +RUSH = http://www.gnu.org.ua/software/rush/manual +rush mono ${RUSH}/rush.html +rush chapter ${RUSH}/html_chapter/ +rush section ${RUSH}/html_section/ +rush node ${RUSH}/html_node/ + +screen mono ${GS}/screen/manual/screen.html +screen node ${GS}/screen/manual/html_node/ + +sed mono ${GS}/sed/manual/sed.html +sed node ${GS}/sed/manual/html_node/ + +sharutils mono ${GS}/sharutils/manual/sharutils.html +sharutils chapter ${GS}/sharutils/manual/html_chapter/ +sharutils node ${GS}/sharutils/manual/html_node/ + +# replaces dmd +shepherd mono ${GS}/shepherd/manual/shepherd.html +shepherd node ${GS}/shepherd/manual/html_node/ + +SMALLTALK = ${GS}/smalltalk +gst mono ${SMALLTALK}/manual/gst.html +gst node ${SMALLTALK}/manual/html_node/ + # + gst-base mono ${SMALLTALK}/manual-base/gst-base.html + gst-base node ${SMALLTALK}/manual-base/html_node/ + # + gst-libs mono ${SMALLTALK}/manual-libs/gst-libs.html + gst-libs node ${SMALLTALK}/manual-libs/html_node/ + +sourceinstall mono ${GS}/sourceinstall/manual/sourceinstall.html +sourceinstall node ${GS}/sourceinstall/manual/html_node/ + +sqltutor mono ${GS}/sqltutor/manual/sqltutor.html +sqltutor node ${GS}/sqltutor/manual/html_node/ + +src-highlite mono ${GS}/src-highlite/source-highlight.html + +swbis mono ${GS}/swbis/manual.html + +tar mono ${GS}/tar/manual/tar.html +tar chapter ${GS}/tar/manual/html_chapter/ +tar section ${GS}/tar/manual/html_section/ +tar node ${GS}/tar/manual/html_node/ + +teseq mono ${GS}/teseq/manual/teseq.html +teseq node ${GS}/teseq/manual/html_node/ + +TEXINFO = ${GS}/texinfo/manual +texinfo mono ${TEXINFO}/texinfo/texinfo.html +texinfo node ${TEXINFO}/texinfo/html_node/ + # + info-stnd mono ${TEXINFO}/info-stnd/info-stnd.html + info-stnd node ${TEXINFO}/info-stnd/html_node/ + # + texi2any_api mono ${TEXINFO}/texi2any_api/texi2any_api.html + texi2any_api node ${TEXINFO}/texi2any_api/html_node/ + # + texi2any_internals mono ${TEXINFO}/texi2any_internals/texi2any_internals.html + texi2any_internals chapter ${TEXINFO}/texi2any_internals/html_chapter/ + +thales node ${GS}/thales/manual/ + +units mono ${GS}/units/manual/units.html +units node ${GS}/units/manual/html_node/ + +vc-dwim mono ${GS}/vc-dwim/manual/vc-dwim.html +vc-dwim node ${GS}/vc-dwim/manual/html_node/ + +wdiff mono ${GS}/wdiff/manual/wdiff.html +wdiff node ${GS}/wdiff/manual/html_node/ + +websocket4j mono ${GS}/websocket4j/manual/websocket4j.html +websocket4j node ${GS}/websocket4j/manual/html_node/ + +wget mono ${GS}/wget/manual/wget.html +wget node ${GS}/wget/manual/html_node/ + +xboard mono ${GS}/xboard/manual/xboard.html +xboard node ${GS}/xboard/manual/html_node/ + +# emacs-page +# Free TeX-related Texinfo manuals on tug.org. + +T = https://tug.org/texinfohtml + +dvipng mono ${T}/dvipng.html +dvips mono ${T}/dvips.html +eplain mono ${T}/eplain.html +kpathsea mono ${T}/kpathsea.html +latex2e mono ${T}/latex2e.html +tlbuild mono ${T}/tlbuild.html +web2c mono ${T}/web2c.html + + +# Local Variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "htmlxrefversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/mu4e/meson.build b/mu4e/meson.build new file mode 100644 index 0000000..a2a22bb --- /dev/null +++ b/mu4e/meson.build @@ -0,0 +1,142 @@ +## Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +# generate some build data for use in mu4e +mu4e_meta = configure_file( + input: 'mu4e-config.el.in', + output: 'mu4e-config.el', + install: true, + install_dir: mu4e_lispdir, + configuration: { + 'VERSION' : meson.project_version(), + 'MU_DOC_DIR' : join_paths(datadir, 'doc', 'mu'), + }) + +mu4e_pkg_desc = configure_file( + input: 'mu4e-pkg.el.in', + output: 'mu4e-pkg.el', + install: true, + install_dir: mu4e_lispdir, + configuration: { + 'VERSION' : meson.project_version(), + 'EMACS_MIN_VERSION' : emacs_min_version, + }) + +mu4e_srcs=[ + 'mu4e-actions.el', + 'mu4e-bookmarks.el', + 'mu4e-compose.el', + 'mu4e-contacts.el', + 'mu4e-context.el', + 'mu4e-contrib.el', + 'mu4e-draft.el', + 'mu4e-folders.el', + 'mu4e.el', + 'mu4e-headers.el', + 'mu4e-helpers.el', + 'mu4e-icalendar.el', + 'mu4e-lists.el', + 'mu4e-main.el', + 'mu4e-mark.el', + 'mu4e-message.el', + 'mu4e-mime-parts.el', + 'mu4e-modeline.el', + 'mu4e-notification.el', + 'mu4e-obsolete.el', + 'mu4e-org.el', + 'mu4e-query-items.el', + 'mu4e-search.el', + 'mu4e-server.el', + 'mu4e-speedbar.el', + 'mu4e-thread.el', + 'mu4e-update.el', + 'mu4e-vars.el', + 'mu4e-view.el', + 'mu4e-window.el' +] + +# note, we cannot compile mu4e-config.el without incurring +# WARNING: Source item +# '[...]/build/mu4e/mu4e-meta.el' cannot be converted to File object, because +# it is a generated file. This will become a hard error in the future. +# +#... so let's not do that! + +foreach src : mu4e_srcs + target_name= '@BASENAME@.elc' + target_path = join_paths(meson.current_build_dir(), target_name) + target_func = '(setq byte-compile-dest-file-function(lambda(_) "' + target_path + '"))' + + # hack-around for native compile issue: copy sources to builddir. + # see: https://debbugs.gnu.org/db/47/47987.html + configure_file(input: src, output:'@BASENAME@.el', copy:true, + install_mode: 'r--r--r--') + + custom_target(src.underscorify() + '_el', + build_by_default: true, + input: src, + output: target_name, + install_dir: mu4e_lispdir, + install: true, + # rebuild all if any changed. + depend_files: mu4e_srcs, + command: [emacs, + '--no-init-file', + '--batch', + '--directory', meson.current_source_dir(), + '--directory', meson.current_build_dir(), + # we don't need warnings for items that have become + # obsolete _after_ our last supported emacs release. + '--eval', '(setq byte-compile-warnings \'(not obsolete))', + '--eval', target_func, + '--funcall', 'batch-byte-compile', '@INPUT@']) + +endforeach + +# this depends on the above hack: all mu4e elisp files needs to be in builddir +mu4e_autoloads = configure_file( + output: 'mu4e-autoloads.el', + install: true, + install_dir: mu4e_lispdir, + command: [emacs, + '--no-init-file', + '--batch', + '--load', 'package', + '--eval', '(package-generate-autoloads "mu4e" "' + + meson.current_build_dir() + '" )']) + +# also install the sources and the config +install_data(mu4e_srcs, install_dir: mu4e_lispdir) + +# install mu4e-about.org +install_data('mu4e-about.org', install_dir : join_paths(datadir, 'doc', 'mu')) + +if makeinfo.found() + custom_target('mu4e_info', + input: 'mu4e.texi', + output: 'mu4e.info', + install_dir: infodir, + install: true, + command: [makeinfo, + '-o', join_paths(meson.current_build_dir(), 'mu4e.info'), + join_paths(meson.current_source_dir(), 'mu4e.texi'), + '-I', join_paths(meson.current_build_dir(), '..')]) + if install_info.found() + infodir = join_paths(get_option('prefix') / get_option('infodir')) + meson.add_install_script(install_info_script, infodir, 'mu4e.info') + endif +endif diff --git a/mu4e/mu4e-about.org b/mu4e/mu4e-about.org new file mode 100644 index 0000000..5c015b3 --- /dev/null +++ b/mu4e/mu4e-about.org @@ -0,0 +1,15 @@ +#+STARTUP:showall +* About mu4e + + *mu4e* is an emacs e-mail client based on the [[http://djcbsoftware.nl/code/mu][mu]] email search engine. It was + written & designed by /Dirk-Jan C. Binnema/, with contributions from others. + + *mu4e* and *mu* are free software, licensed under the terms of the [[http://www.gnu.org/licenses/gpl-3.0.html][GNU GPLv3]]. + + You can get the code from [[https://github.com/djcb/mu][the git repository]]; there, you can also + [[https://github.com/djcb/mu/issues][file bugs and feature requests]]. + + *mu4e* has its own [[info:mu4e][manual]], which includes an [[info:mu4e#FAQ%20-%20Frequently%20Anticipated%20Questions][FAQ]]. If that is not enough, + there's also the [[http://groups.google.com/group/mu-discuss][mu mailing list]]. + + [Press *q* to quit this buffer] diff --git a/mu4e/mu4e-actions.el b/mu4e/mu4e-actions.el new file mode 100644 index 0000000..543ebac --- /dev/null +++ b/mu4e/mu4e-actions.el @@ -0,0 +1,275 @@ +;;; mu4e-actions.el --- Actions for messages/attachments -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Example actions for messages, attachments (see chapter 'Actions' in the +;; manual) + +;;; Code: + +(require 'ido) +(require 'browse-url) + +(require 'mu4e-helpers) +(require 'mu4e-message) +(require 'mu4e-search) +(require 'mu4e-contacts) +(require 'mu4e-lists) + +;;; Count lines + +(defun mu4e-action-count-lines (msg) + "Count the number of lines in the e-mail MSG. +Works for headers view and message-view." + (message "Number of lines: %s" + (shell-command-to-string + (concat "wc -l < " + (shell-quote-argument (mu4e-message-field msg :path)))))) + +;;; Org Helpers + +(defvar mu4e-captured-message nil + "The most recently captured message.") + +(defun mu4e-action-capture-message (msg) + "Remember MSG. +Later, we can create an attachment based on this message with +`mu4e-compose-attach-captured-message'." + (setq mu4e-captured-message msg) + (message "Message has been captured")) + + +(defun mu4e-action-copy-message-file-path (msg) + "Save the full path for the current MSG to the kill ring." + (kill-new (mu4e-message-field msg :path))) + +(defvar mu4e-org-contacts-file nil + "File to store contact information for org-contacts. +Needed by `mu4e-action-add-org-contact'.") + +(eval-when-compile ;; silence compiler warning about free variable + (unless (require 'org-capture nil 'noerror) + (defvar org-capture-templates nil))) + +(defun mu4e-action-add-org-contact (msg) + "Add an org-contact based on the sender ddress of the current MSG. +You need to set `mu4e-org-contacts-file' to the full path to the +file where you store your org-contacts." + (unless (require 'org-capture nil 'noerror) + (mu4e-error "Feature org-capture is not available")) + (unless mu4e-org-contacts-file + (mu4e-error "Variable `mu4e-org-contacts-file' is nil")) + (let* ((sender (car-safe (mu4e-message-field msg :from))) + (name (mu4e-contact-name sender)) + (email (mu4e-contact-email sender)) + (blurb + (format + (concat + "* %%?%s\n" + ":PROPERTIES:\n" + ":EMAIL: %s\n" + ":NICK:\n" + ":BIRTHDAY:\n" + ":END:\n\n") + (or name email "") + (or email ""))) + (key "mu4e-add-org-contact-key") + (org-capture-templates + (append org-capture-templates + (list (list key "contacts" 'entry + (list 'file mu4e-org-contacts-file) blurb))))) + (when (fboundp 'org-capture) + (org-capture nil key)))) + +;;; Patches + +(defvar mu4e--patch-directory-history nil + "History of directories we have applied patches to.") + +;; This essentially works around the fact that read-directory-name +;; can't have custom history. +(defun mu4e--read-patch-directory (&optional prompt) + "Read a `PROMPT'ed directory name via `completing-read' with history." + (unless prompt + (setq prompt "Target directory:")) + (file-truename + (completing-read prompt 'read-file-name-internal #'file-directory-p + nil nil 'mu4e--patch-directory-history))) + +(defun mu4e-action-git-apply-patch (msg) + "Apply `MSG' as a git patch." + (let ((path (mu4e--read-patch-directory "Target directory: "))) + (let ((default-directory path)) + (shell-command + (format "git apply %s" + (shell-quote-argument (mu4e-message-field msg :path))))))) + +(defun mu4e-action-git-apply-mbox (msg &optional signoff) + "Apply `MSG' a git patch with optional `SIGNOFF'. + +If the `default-directory' matches the most recent history entry don't +bother asking for the git tree again (useful for bulk actions)." + + (let ((cwd (substring-no-properties + (or (car mu4e--patch-directory-history) + "not-a-dir")))) + (unless (and (stringp cwd) (string= default-directory cwd)) + (setq cwd (mu4e--read-patch-directory "Target directory: "))) + (let ((default-directory cwd)) + (shell-command + (format "git am %s %s" + (if signoff "--signoff" "") + (shell-quote-argument (mu4e-message-field msg :path))))))) + +;;; Tagging + +(defvar mu4e-action-tags-header "X-Keywords" + "Header where tags are stored. +Used by `mu4e-action-retag-message'. Make sure it is one of the +headers mu recognizes for storing tags: X-Keywords, X-Label, +Keywords. Also note that changing this setting on already tagged +messages can lead to messages with multiple tags headers.") + +(defvar mu4e-action-tags-completion-list '() + "List of tags for completion in `mu4e-action-retag-message'.") + +(defun mu4e--contains-line-matching (regexp path) + "Return non-nil if the file at PATH contain a line matching REGEXP. +Otherwise return nil." + (with-temp-buffer + (insert-file-contents path) + (save-excursion + (goto-char (point-min)) + (re-search-forward regexp nil t)))) + +(defun mu4e--replace-first-line-matching (regexp to-string path) + "Replace first line matching REGEXP in PATH with TO-STRING." + (with-temp-file path + (insert-file-contents path) + (save-excursion + (goto-char (point-min)) + (if (re-search-forward regexp nil t) + (replace-match to-string t nil))))) + +(declare-function mu4e--server-add "mu4e-server") +(defun mu4e--refresh-message (path) + "Re-parse message at PATH. +if this works, we will +receive (:info add :path <path> :docid <docid>) as well as (:update +<msg-sexp>)." + (mu4e--server-add path)) + +(defun mu4e-action-retag-message (msg &optional retag-arg) + "Change tags of MSG with RETAG-ARG. + +RETAG-ARG is a comma-separated list of additions and removals. + +Example: +tag,+long tag,-oldtag +would add \"tag\" and \"long tag\", and remove \"oldtag\"." + (let* ( + (path (mu4e-message-field msg :path)) + (oldtags (mu4e-message-field msg :tags)) + (tags-completion + (append + mu4e-action-tags-completion-list + (mapcar (lambda (tag) (format "+%s" tag)) + mu4e-action-tags-completion-list) + (mapcar (lambda (tag) (format "-%s" tag)) + oldtags))) + (retag (if retag-arg + (split-string retag-arg ",") + (completing-read-multiple "Tags: " tags-completion))) + (header mu4e-action-tags-header) + (sep (cond ((string= header "Keywords") ", ") + ((string= header "X-Label") " ") + ((string= header "X-Keywords") ", ") + (t ", "))) + (taglist (if oldtags (copy-sequence oldtags) '())) + tagstr) + (dolist (tag retag taglist) + (cond + ((string-match "^\\+\\(.+\\)" tag) + (setq taglist (push (match-string 1 tag) taglist))) + ((string-match "^\\-\\(.+\\)" tag) + (setq taglist (delete (match-string 1 tag) taglist))) + (t + (setq taglist (push tag taglist))))) + + (setq taglist (sort (delete-dups taglist) 'string<)) + (setq tagstr (mapconcat 'identity taglist sep)) + + (setq tagstr (replace-regexp-in-string "[\\&]" "\\\\\\&" tagstr)) + (setq tagstr (replace-regexp-in-string "[/]" "\\&" tagstr)) + + (if (not (mu4e--contains-line-matching (concat header ":.*") path)) + ;; Add tags header just before the content + (mu4e--replace-first-line-matching + "^$" (concat header ": " tagstr "\n") path) + + ;; replaces keywords, restricted to the header + (mu4e--replace-first-line-matching + (concat header ":.*") + (concat header ": " tagstr) + path)) + + (mu4e-message (concat "tagging: " (mapconcat 'identity taglist ", "))) + (mu4e--refresh-message path))) + +(defun mu4e-action-show-thread (msg) + "Show thread for message at point with point remaining on MSG. +I.e., point remains on the message with the message-id where the +action was invoked. If invoked in view mode, continue to display +the message." + (let ((msgid (mu4e-message-field msg :message-id))) + (when msgid + (let ((mu4e-search-threads t) + (mu4e-search-include-related t)) + (mu4e-search + (format "msgid:%s" msgid) + nil nil nil + msgid (and (eq major-mode 'mu4e-view-mode) + (not (eq mu4e-split-view 'single-window)))))))) + + +;;; Mailing list URLS + +(defun mu4e-action-browse-list-archive (msg) + "Browse the archive for a mailing list message MSG. +See `mu4e-mailing-list-archive-url'." + (interactive (list (mu4e-message-at-point))) + (if-let ((url (mu4e-mailing-list-archive-url msg))) + (browse-url url) + (mu4e-warn "No archive available for this message"))) + +(defun mu4e-action-copy-list-archive-url (msg) + "Copy the archive url for a mailing list message MSG. +See `mu4e-mailing-list-archive-url'." + (interactive (list (mu4e-message-at-point))) + (let ((url (mu4e-mailing-list-archive-url msg))) + (if (stringp url) + (kill-new url) + (mu4e-warn "Cannot get archive URL for this message")))) + +;;; +(provide 'mu4e-actions) +;;; mu4e-actions.el ends here diff --git a/mu4e/mu4e-bookmarks.el b/mu4e/mu4e-bookmarks.el new file mode 100644 index 0000000..452169b --- /dev/null +++ b/mu4e/mu4e-bookmarks.el @@ -0,0 +1,195 @@ +;;; mu4e-bookmarks.el --- Bookmarks handling -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: +(require 'mu4e-helpers) +(require 'mu4e-modeline) +(require 'mu4e-folders) +(require 'mu4e-query-items) + + +;;; Configuration + +(defgroup mu4e-bookmarks nil + "Settings for bookmarks." + :group 'mu4e) + +(defcustom mu4e-bookmarks + '(( :name "Unread messages" + :query "flag:unread AND NOT flag:trashed" + :key ?u) + ( :name "Today's messages" + :query "date:today..now" + :key ?t) + ( :name "Last 7 days" + :query "date:7d..now" + :hide-unread t + :key ?w) + ( :name "Messages with images" + :query "mime:image/*" + :key ?p)) + "List of pre-defined queries that are shown on the main screen. + +Each of the list elements is a plist with at least: +`:name' - the name of the query +`:query' - the query expression string or function +`:key' - the shortcut key (single character) + +Optionally, you can add the following: + +- `:favorite' - if t, monitor the results of this query, and make +it eligible for showing its status in the modeline. At most +one bookmark should have this set to t (otherwise the _first_ +bookmark is the implicit favorite). The query for the `:favorite' +item must be unique among `mu4e-bookmarks' and +`mu4e-maildir-shortcuts'. +- `:hide' - if t, the bookmark is hidden from the main-view and +speedbar. +- `:hide-unread' - do not show the counts of +unread/total number of matches for the query in the main-view. +This can be useful if a bookmark uses a very slow query. + +`:hide-unread' is implied from `:hide'. + +Note: for efficiency, queries used to determine the unread/all +counts do not discard duplicate or unreadable messages. Thus, the +numbers shown may differ from the number you get from a normal +query." + :type '(repeat (plist)) + :group 'mu4e-bookmarks) + + +(defun mu4e-ask-bookmark (prompt) + "Ask user for bookmark using PROMPT. +Return the corresponding query. The bookmark are as defined in +`mu4e-bookmarks'." + (unless (mu4e-bookmarks) (mu4e-error "No bookmarks defined")) + (let* ((bmarks (seq-map (lambda (bm) + (cons (format "%c%s" + (plist-get bm :key) + (plist-get bm :name)) + (plist-get bm :query))) + (mu4e-filter-single-key (mu4e-bookmarks))))) + (mu4e-read-option prompt bmarks))) + +(defun mu4e-get-bookmark-query (kar) + "Get the corresponding bookmarked query for shortcut KAR. +Raise an error if none is found." + (let ((chosen-bm + (or (seq-find + (lambda (bm) + (= kar (plist-get bm :key))) + (mu4e-bookmarks)) + (mu4e-warn "Unknown shortcut '%c'" kar)))) + (mu4e--bookmark-query chosen-bm))) + +(defun mu4e-bookmark-define (query name key) + "Define a bookmark for QUERY with NAME and shortcut KEY. +Append it to `mu4e-bookmarks'. Replaces any existing bookmark +with KEY." + (setq mu4e-bookmarks + (seq-remove + (lambda (bm) + (= (plist-get bm :key) key)) + (mu4e-bookmarks))) + (cl-pushnew `(:name ,name + :query ,query + :key ,key) + mu4e-bookmarks :test 'equal)) + +(defun mu4e-bookmarks () + "Get `mu4e-bookmarks' in the (new) format. +Convert from the old format if needed." + (seq-map (lambda (item) + (if (and (listp item) (= (length item) 3)) + `(:name ,(nth 1 item) :query ,(nth 0 item) + :key ,(nth 2 item)) + item)) + mu4e-bookmarks)) + +(defun mu4e-bookmark-favorite () + "Find the favorite bookmark." + ;; note, use query-items, which will have picked a favorite + ;; even if user did not provide one explictly + (seq-find + (lambda (item) + (plist-get item :favorite)) + (mu4e-query-items 'bookmarks))) + +;; for Zero-Inbox afficionados +(defvar mu4e-modeline-all-clear '("C:" . "🌀") + "No more messages at all for this query.") +(defvar mu4e-modeline-all-read '("R:" . "✅") + "No unread messages left.") +(defvar mu4e-modeline-unread-items '("U:" . "📫") + "There are some unread items.") +(defvar mu4e-modeline-new-items '("N:" . "🔥") + "There are some new items after the baseline. +I.e., very new messages.") + +(declare-function mu4e-search-bookmark "mu4e-search") +(defun mu4e-jump-to-favorite () + "Jump to to the favorite bookmark, if any." + (interactive) + (when-let ((fav (mu4e--bookmark-query (mu4e-bookmark-favorite)))) + (mu4e-search-bookmark fav))) + +(defun mu4e--bookmarks-modeline-item () + "Modeline item showing message counts for the favorite bookmark. + +This uses the one special ':favorite' bookmark, and if there is +one, creates a propertized string for display in the modeline." + (when-let ((fav ;; any results for the favorite bookmark item? + (seq-find (lambda (bm) (plist-get bm :favorite)) + (mu4e-query-items 'bookmarks)))) + (cl-destructuring-bind (&key unread count delta-unread + &allow-other-keys) fav + (propertize + (format "%s%s " + (funcall (if mu4e-use-fancy-chars 'cdr 'car) + (cond + ((> delta-unread 0) mu4e-modeline-new-items) + ((> unread 0) mu4e-modeline-unread-items) + ((> count 0) mu4e-modeline-all-read) + (t mu4e-modeline-all-clear))) + (mu4e--query-item-display-counts fav)) + 'help-echo + (format + (concat + "mu4e favorite bookmark '%s':\n" + "\t%s\n\n" + "number of matches: %d\n" + "unread messages: %d\n" + "changes since baseline: %+d\n") + (plist-get fav :name) + (mu4e--bookmark-query fav) + count unread delta-unread) + 'mouse-face 'mode-line-highlight + 'keymap '(mode-line keymap + (mouse-1 . mu4e-jump-to-favorite) + (mouse-2 . mu4e-jump-to-favorite) + (mouse-3 . mu4e-jump-to-favorite)))))) + +(provide 'mu4e-bookmarks) +;;; mu4e-bookmarks.el ends here diff --git a/mu4e/mu4e-compose.el b/mu4e/mu4e-compose.el new file mode 100644 index 0000000..6135ef5 --- /dev/null +++ b/mu4e/mu4e-compose.el @@ -0,0 +1,521 @@ +;;; mu4e-compose.el --- Compose and send messages -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Implements mu4e-compose-mode, which is a `message-mode' derivative. There's +;; quite a bit of trickery involved to make the message-mode functions work in +;; this context; see mu4e-draft for details. + + +;;; Code: +(require 'message) +(require 'sendmail) +(require 'gnus-msg) +(require 'nnheader) ;; for make-full-mail-header + +(require 'mu4e-obsolete) +(require 'mu4e-server) +(require 'mu4e-message) +(require 'mu4e-context) +(require 'mu4e-folders) + +(require 'mu4e-draft) + + +;;; User configuration for compose-mode +(defgroup mu4e-compose nil + "Customization for composing/sending messages." + :group 'mu4e) + +(defcustom mu4e-compose-format-flowed nil + "Whether to compose messages to be sent as format=flowed. +\(Or with long lines if variable `use-hard-newlines' is set to +nil). The variable `fill-flowed-encode-column' lets you customize +the width beyond which format=flowed lines are wrapped." + :type 'boolean + :safe 'booleanp + :group 'mu4e-compose) + +(defcustom mu4e-compose-pre-hook nil + "Hook run just *before* message composition starts. + +If the compose-type is a symbol, either `reply' or `forward', the +variable `mu4e-compose-parent-message' is the message replied to +/ being forwarded / edited, and `mu4e-compose-type' contains the +type of message to be composed. + +Note that there is no draft message yet when this hook runs, it +is meant for influencing the how mu4e constructs the draft +message. If you want to do something with the draft messages +after it has been constructed, `mu4e-compose-mode-hook' would be +the place to do that." + :type 'hook + :group 'mu4e-compose) + +(defcustom mu4e-compose-post-hook + (list + ;; kill compose frames + #'mu4e-compose-post-kill-frame + ;; attempt to restore the old configuration. + #'mu4e-compose-post-restore-window-configuration) + "Hook run *after* message composition is over. + +This is hook is run when closing the composition buffer, either +by sending, postponing, exiting or killing it. + +This multiplexes the `message-mode' hooks `message-send-actions', +`message-postpone-actions', `message-exit-actions' and +`message-kill-actions', and the hook is run with a variable +`mu4e-compose-post-trigger' set correspondingly to a symbol, +`send', `postpone', `exit' or `kill'." + :type 'hook + :group 'mu4e-compose) + + + +(defvar mu4e-captured-message) +(defun mu4e-compose-attach-captured-message () + "Insert the last captured message file as an attachment. +Messages are captured with `mu4e-action-capture-message'." + (interactive) + (if-let* ((msg mu4e-captured-message) + (path (plist-get msg :path)) + (path (and (file-exists-p path) path))) + (mml-attach-file + path + "message/rfc822" + (or (plist-get msg :subject) "No subject") + "attachment") + (mu4e-warn "No valid message has been captured"))) + +;; Go to bottom / top + +(defun mu4e-compose-goto-top (&optional arg) + "Go to the beginning of the message or buffer. +Go to the beginning of the message or, if already there, go to +the beginning of the buffer. + +Push mark at previous position, unless either a +\\[universal-argument] prefix ARG is supplied, or Transient Mark mode +is enabled and the mark is active." + (interactive "P") + (or arg + (region-active-p) + (push-mark)) + (let ((old-position (point))) + (message-goto-body) + (when (equal (point) old-position) + (goto-char (point-min))))) + +(defun mu4e-compose-goto-bottom (&optional arg) + "Go to the end of the message or buffer. +Go to the end of the message (before signature) or, if already +there, go to the end of the buffer. + +Push mark at previous position, unless either a +\\[universal-argument] prefix ARG is supplied, or Transient Mark mode +is enabled and the mark is active." + (interactive "P") + (or arg + (region-active-p) + (push-mark)) + (let ((old-position (point)) + (message-position (save-excursion (message-goto-body) (point)))) + (goto-char (point-max)) + (when (re-search-backward message-signature-separator message-position t) + (forward-line -1)) + (when (equal (point) old-position) + (goto-char (point-max))))) + +(defun mu4e-compose-context-switch (&optional force name) + "Change the context for the current draft message. + +With NAME, switch to the context with NAME, and with FORCE non-nil, +switch even if the switch is to the same context. + +Like `mu4e-context-switch' but with some changes after switching: +1. Update the From and Organization headers as per the new context +2. Update the `message-signature' as per the new context. + +Unlike some earlier version of this function, does _not_ update +the draft folder for the messages, as that would require changing +the file under our feet, which is a bit fragile." + (interactive "P") + + (unless (derived-mode-p 'mu4e-compose-mode) + (mu4e-error "Only available in mu4e compose buffers")) + + (let ((old-context (mu4e-context-current))) + (unless (and name (not force) (eq old-context name)) + (unless (and (not force) + (eq old-context (mu4e-context-switch nil name))) + (save-excursion + ;; Change From / Organization if needed. + (message-replace-header "Organization" + (or (message-make-organization) "") + '("Subject")) ;; keep in same place + (message-replace-header "From" + (or (message-make-from) "")) + ;; Update signature. + (when (message-goto-signature) ;; delete old signature. + (if message-signature-insert-empty-line + (forward-line -2) (forward-line -1)) + (delete-region (point) (point-max))) + (when message-signature + (save-excursion (message-insert-signature)))))))) + + +;;; address completion + +;; inspired by org-contacts.el and +;; https://github.com/nordlow/elisp/blob/master/mine/completion-styles-cycle.el + +(defun mu4e--compose-complete-handler (str pred action) + "Complete address STR with predication PRED for ACTION." + (cond + ((eq action nil) + (try-completion str mu4e--contacts-set pred)) + ((eq action t) + (all-completions str mu4e--contacts-set pred)) + ((eq action 'metadata) + ;; our contacts are already sorted - just need to tell the completion + ;; machinery not to try to undo that... + '(metadata + (display-sort-function . identity) + (cycle-sort-function . identity))))) + +(defun mu4e-complete-contact () + "Attempt to complete the text at point with a contact. +I.e., either \"name <email>\" or \"email\". Return nil if not found. + +This function can be used for `completion-at-point-functions', to +complete addresses. This can be used from outside mu4e, but mu4e +must be active (running) for this to work." + (let* ((end (point)) + (start (save-excursion + (re-search-backward "\\(\\`\\|[\n:,]\\)[ \t]*") + (goto-char (match-end 0)) + (point)))) + (list start end #'mu4e--compose-complete-handler))) + +(defun mu4e--compose-complete-contact-field () + "Attempt to complete a contact when in a contact field. + +This is like `mu4e-compose-complete-contact', but limited to the +contact fields." + (let ((mail-abbrev-mode-regexp + "^\\(To\\|B?Cc\\|Reply-To\\|From\\|Sender\\):") + (mail-header-separator mu4e--header-separator)) + (when (mail-abbrev-in-expansion-header-p) + (mu4e-complete-contact)))) + +(defun mu4e--compose-setup-completion () + "Maybe enable auto-completion of addresses. +Do this when `mu4e-compose-complete-addresses' is non-nil. + +When enabled, this attempts to put mu4e's completions at the +start of the buffer-local `completion-at-point-functions'. Other +completion functions still apply." + (when mu4e-compose-complete-addresses + (set (make-local-variable 'completion-ignore-case) t) + (set (make-local-variable 'completion-cycle-threshold) 7) + (add-to-list (make-local-variable 'completion-styles) 'substring) + (add-hook 'completion-at-point-functions + #'mu4e--compose-complete-contact-field -10 t))) + + ;;; mu4e-compose-mode +(defun mu4e--compose-remap-faces () + "Remap `message-mode' faces to mu4e ones. + +Our parent `message-mode' uses font-locking for the compose +buffers; lets remap its faces so it uses the ones for mu4e." + ;; normal headers + (face-remap-add-relative 'message-header-name 'mu4e-header-field-face) + (face-remap-add-relative 'message-header-other 'mu4e-header-value-face) + ;; special headers + (face-remap-add-relative 'message-header-from 'mu4e-contact-face) + (face-remap-add-relative 'message-header-to 'mu4e-contact-face) + (face-remap-add-relative 'message-header-cc 'mu4e-contact-face) + (face-remap-add-relative 'message-header-bcc 'mu4e-contact-face) + (face-remap-add-relative 'message-header-subject + 'mu4e-special-header-value-face)) + +(defvar mu4e-compose-mode-map + (let ((map (make-sparse-keymap))) + (set-keymap-parent map message-mode-map) + (define-key map (kbd "C-S-u") #'mu4e-update-mail-and-index) + (define-key map (kbd "C-c C-u") #'mu4e-update-mail-and-index) + (define-key map (kbd "C-c ;") #'mu4e-compose-context-switch) + + ;; emacs 29 + ;;(keymap-set map "<remap> <beginning-of-buffer>" #'mu4e-compose-goto-top) + ;;(keymap-set map "<remap> <end-of-buffer>" #'mu4e-compose-goto-bottom) + (define-key map (vector 'remap #'beginning-of-buffer) + #'mu4e-compose-goto-top) + (define-key map (vector 'remap #'end-of-buffer) + #'mu4e-compose-goto-bottom) + + ;; remove some unsupported commands... [remap ..] does not work here + ;; XXX remove from menu, too. + (define-key map (kbd "C-c C-f C-n") nil) ;; message-goto-newsgroups + (define-key map (kbd "C-c C-n") nil) ;; message-insert-newsgroups + (define-key map (kbd "C-c C-j") nil) ;; gnus-delay-article + map) + "The keymap for mu4e-compose buffers.") + +(defun mu4e--compose-unsupported (&rest _args) + "Advise wrapper for Gnus unsupported functions in mu4e." + (when (eq major-mode 'mu4e-compose-mode) + (mu4e-warn "Not available in mu4e"))) + +(defun mu4e--neutralize-undesirables () + "Beware Gnus commands that do not work with mu4e." + ;; the Field menu contains many items that don't apply. + (advice-add 'gnus-delay-article + :before #'mu4e--compose-unsupported) ;; # XXX does not work?! + (advice-add 'message-goto-newsgroups :before #'mu4e--compose-unsupported) + (advice-add 'message-insert-newsgroups :before #'mu4e--compose-unsupported)) + +(define-derived-mode mu4e-compose-mode message-mode "mu4e:compose" + "Major mode for the mu4e message composition, derived from `message-mode'. +\\{mu4e-compose-mode-map}." + (progn + (use-local-map mu4e-compose-mode-map) + (mu4e-context-minor-mode) + (mu4e--neutralize-undesirables) + (mu4e--compose-remap-faces) + (setq-local nobreak-char-display nil) + ;; set this to allow mu4e to work when gnus-agent is unplugged in gnus + (set (make-local-variable 'message-send-mail-real-function) nil) + ;; Set to nil to enable `electric-quote-local-mode' to work: + (set (make-local-variable 'comment-use-syntax) nil) + (mu4e--compose-setup-completion) ;; maybe offer address completion + (if mu4e-compose-format-flowed ;; format-flowed + (progn + (turn-off-auto-fill) + (setq truncate-lines nil + word-wrap t + mml-enable-flowed t + use-hard-newlines t) + (visual-line-mode t)) + (setq mml-enable-flowed nil)))) + +(declare-function mu4e-view-message-text "mu4e-view") + +(defun mu4e-message-cite-nothing () + "Function for `message-cite-function' that cites _nothing_." + (save-excursion + (message-cite-original-without-signature) + (delete-region (point-min) (point-max)))) + +(defun mu4e--compose-cite (msg) + "Return a cited version of the ORIG message MSG (a string). +This function uses `message-cite-function', and its settings apply." + (with-temp-buffer + (insert (mu4e-view-message-text msg)) + (goto-char (point-min)) + (push-mark (point-max)) + (let ((message-signature-separator "^-- *$") + (message-signature-insert-empty-line t)) + (funcall message-cite-function)) + (pop-mark) + (goto-char (point-min)) + (buffer-string))) + + +;;;###autoload +(defalias 'mu4e-compose-mail #'mu4e-compose-new) + +;;;###autoload +(defun mu4e-compose-new (&optional to subject other-headers continue + _switch-function yank-action send-actions + return-action &rest _) + "Mu4e's implementation of `compose-mail'. +TO, SUBJECT, OTHER-HEADERS, CONTINUE, YANK-ACTION SEND-ACTIONS +RETURN-ACTION are as described in `compose-mail', and to the +extend that they do not conflict with mu4e's inner workings. +SWITCH-FUNCTION is ignored." + (interactive) + (mu4e--draft + 'new + (lambda () (mu4e--message-call + #'message-mail to subject other-headers continue + nil ;; switch-function -> we handle it ourselves. + yank-action send-actions return-action)))) + +;;;###autoload +(defun mu4e-compose-reply-to (&optional to wide) + "Reply to the message at point. +Optional TO can be the To: address for the message. If WIDE is +non-nil, make it a \"wide\" reply (a.k.a. \"reply-to-all\")." + (interactive) + (let ((parent (mu4e-message-at-point))) + (mu4e--draft-with-parent + 'reply parent + (lambda () + (with-current-buffer (mu4e--message-call #'message-reply to wide) + (message-goto-body) + (insert (mu4e--compose-cite parent)) + (current-buffer)))))) + +;;;###autoload +(defun mu4e-compose-reply (&optional wide) + "Reply to the message at point. If WIDE is +non-nil, make it a \"wide\" reply (a.k.a. \"reply-to-all\")." + (interactive "P") + (mu4e-compose-reply-to nil wide)) + +;;;###autoload +(defun mu4e-compose-wide-reply () + "Wide reply to the message at point. +(a.k.a. \"reply-to-all\")." + (interactive) + (mu4e-compose-reply-to nil t))1 + +;;;###autoload +(defun mu4e-compose-supersede () + "Supersede the message at point. + +That is, send the message again, with all the same recipients; +this can be useful to follow-up on a sent message. The message +must originate from the current user, as determined through +`mu4e-personal-or-alternative-address-p'." + (interactive) + (let ((parent (mu4e-message-at-point))) + (mu4e--draft-with-parent + 'reply ;; it's a special kind of reply. + parent + (lambda () + (with-current-buffer (mu4e--message-call #'message-supersede)))))) + +(defun mu4e-compose-forward () + "Forward the message at point. +To influence the way a message is forwarded, you can use the +variables ‘message-forward-as-mime’ and +‘message-forward-show-mml’." + (interactive) + (let ((parent (mu4e-message-at-point))) + (mu4e--draft-with-parent + 'forward parent + (lambda () + (setq + message-reply-headers (make-full-mail-header + 0 + (or (message-field-value "Subject") "none") + (or (message-field-value "From") "nobody") + (message-field-value "Date") + (message-field-value "Message-Id" t) + (message-field-value "References") + 0 0 "")) + ;; a bit of a hack; mu4e--draft-with-parent will insert the decoded + ;; version of the message, but that's not good enough for + ;; message-forward, which needs the raw message instead; see #2662. + (erase-buffer) + (insert-file-contents-literally (mu4e-message-readable-path parent)) + (with-current-buffer (mu4e--message-call #'message-forward) + (current-buffer)))))) + +;;;###autoload +(defun mu4e-compose-edit() + "Edit an existing draft message." + (interactive) + (let* ((msg (mu4e-message-at-point))) + (unless (member 'draft (mu4e-message-field msg :flags)) + (mu4e-warn "Cannot edit non-draft messages")) + (mu4e--draft + 'edit + (lambda () + (with-current-buffer + (find-file-noselect (mu4e-message-readable-path msg)) + (mu4e--delimit-headers) + (current-buffer)))))) + +;;;###autoload +(defun mu4e-compose-resend (address) + "Re-send the message at point to ADDRESS. +The message is resent as-is, without any editing. See +`message-resend' for details." + (interactive + (list (completing-read + "Resend message to address: " mu4e--contacts-set))) + (let ((msg (mu4e-message-at-point))) + (with-temp-buffer + (mu4e--prepare-draft msg) + (insert-file-contents (mu4e-message-readable-path msg)) + (message-resend address)))) + +;;; Compose Mail + +(declare-function mu4e "mu4e") + +;;;###autoload +(define-mail-user-agent 'mu4e-user-agent + #'mu4e-compose-mail + #'message-send-and-exit + #'message-kill-buffer + 'message-send-hook) + +;; Without this, `mail-user-agent' cannot be set to `mu4e-user-agent' +;; through customize, as the custom type expects a function. Not +;; sure whether this function is actually ever used; if it is then +;; returning the symbol is probably the correct thing to do, as other +;; such functions suggest. +(defun mu4e-user-agent () + "Return the `mu4e-user-agent' symbol." + 'mu4e-user-agent) + +;;; minor mode for use in other modes. +(defvar mu4e-compose-minor-mode-map + (let ((map (make-sparse-keymap))) + (define-key map "R" #'mu4e-compose-reply) + (define-key map "W" #'mu4e-compose-wide-reply) + (define-key map "F" #'mu4e-compose-forward) + (define-key map "E" #'mu4e-compose-edit) + (define-key map "C" #'mu4e-compose-new) + map) + "Keymap for compose minor-mode.") + +(define-minor-mode mu4e-compose-minor-mode + "Mode for searching for messages." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap mu4e-compose-minor-mode-map) + +(defvar mu4e--compose-menu-items + '("--" + ["Compose new" mu4e-compose-new + :help "Compose new message"] + ["Reply" mu4e-compose-reply + :help "Reply to message"] + ["Reply to all" mu4e-compose-wide-reply + :help "Reply to all-recipients"] + ["Forward" mu4e-compose-forward + :help "Forward message"] + ["Resend" mu4e-compose-resend + :help "Re-send message"]) + "Easy menu items for message composition.") + ;;; +(provide 'mu4e-compose) +;;; mu4e-compose.el ends here diff --git a/mu4e/mu4e-config.el.in b/mu4e/mu4e-config.el.in new file mode 100644 index 0000000..5f99db4 --- /dev/null +++ b/mu4e/mu4e-config.el.in @@ -0,0 +1,9 @@ +;; auto-generated + +(defconst mu4e-mu-version "@VERSION@" + "Required mu binary version; mu4e's version must agree with this.") + +(defconst mu4e-doc-dir "@MU_DOC_DIR@" + "Mu4e's data-dir.") + +(provide 'mu4e-config) diff --git a/mu4e/mu4e-contacts.el b/mu4e/mu4e-contacts.el new file mode 100644 index 0000000..ab6079c --- /dev/null +++ b/mu4e/mu4e-contacts.el @@ -0,0 +1,308 @@ +;;; mu4e-contacts.el --- Dealing with contacts -*- lexical-binding: t -*- + +;; Copyright (C) 2022-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Utility functions used in the mu4e + +;;; Code: +(require 'cl-lib) +(require 'message) +(require 'mu4e-helpers) +(require 'mu4e-update) + + +;;; Configuration +(defcustom mu4e-compose-complete-addresses t + "Whether to do auto-completion of e-mail addresses." + :type 'boolean + :group 'mu4e-compose) + +(defcustom mu4e-compose-complete-only-personal nil + "Whether to consider only \"personal\" e-mail addresses for completion. +That is, addresses from messages where user was explicitly in one +of the address fields (this excludes mailing list messages). +These addresses are the ones specified with \"mu init\"." + :type 'boolean + :group 'mu4e-compose) + +(defcustom mu4e-compose-complete-only-after "2018-01-01" + "Consider only contacts last seen after this date. + +Date must be a string of the form YYYY-MM-DD. + +This is useful for limiting a potentially enormous set of +contacts for auto-completion to just those that are present in +the e-mail corpus in recent times. Set to nil to not have any +time-based restriction." + :type 'string + :group 'mu4e-compose) + +(defcustom mu4e-compose-complete-max nil + "Limit the amount of contacts for completion, nil for no limits. +After considering the other constraints +\(`mu4e-compose-complete-addresses' and +`mu4e-compose-complete-only-after'), pick only the highest-ranked +<n>. + +Lowering this variable reduces start-up time and memory usage." + :type '(choice natnum (const :tag "No limits" nil)) + :group 'mu4e-compose) + +;; names and mail-addresses can be mapped onto their canonical +;; counterpart. use the customizeable function +;; mu4e-canonical-contact-function to do that. below the identity +;; function for mapping a contact onto the canonical one. +(defun mu4e-contact-identity (contact) + "Return the name and the mail-address of a CONTACT. +It is used as the identity function for converting contacts to +their canonical counterpart; useful as an example." + (let ((name (plist-get contact :name)) + (mail (plist-get contact :mail))) + (list :name name :mail mail))) + +(defcustom mu4e-contact-process-function + (lambda(addr) + (cond + ((string-match-p "reply" addr) + ;; no-reply adresses are not useful of course, but neither are are + ;; reply-xxxx addresses since they're autogenerated only useful for direct + ;; replies. + nil) + (t addr))) + "Function for processing contact information for use in auto-completion. + +The function receives the contact as a string, e.g \"Foo Bar + <foo.bar@example.com>\" \"cuux@example.com\" + +The function should return either: +- nil: do not use this contact for completion +- the (possibly rewritten) address, which must be +an RFC-2822-compatible e-mail address." + :type 'function + :group 'mu4e-compose) + +(defcustom mu4e-compose-reply-ignore-address + '("no-?reply") + "Addresses to prune when doing wide replies. + +This can be a regexp matching the address, a list of regexps or a +predicate function. A value of nil keeps all the addresses." + :type '(choice + (const nil) + function + string + (repeat string)) + :group 'mu4e-compose) + + +;;; Internal variables +(defvar mu4e--contacts-tstamp "0" + "Timestamp for the most recent contacts update." ) + +(defvar mu4e--contacts-set nil + "Set with the full contact addresses for autocompletion.") + +;;; user mail address +(defun mu4e-personal-addresses (&optional no-regexp) + "Get the list user's personal addresses, as passed to \"mu init\". + +The address are either plain e-mail addresses or regexps (strings + wrapped / /). When NO-REGEXP is non-nil, do not include regexp + address patterns (if any)." + (seq-remove + (lambda (addr) (and no-regexp (string-match-p "^/.*/" addr))) + (when-let ((props (mu4e-server-properties))) + (plist-get props :personal-addresses)))) + +(defun mu4e-personal-address-p (addr) + "Is ADDR a personal address? +Evaluate to nil if ADDR does not match any of the personal +addresses. Uses \\=(mu4e-personal-addresses) for the addresses +with both the plain addresses and /regular expressions/." + (when addr + (seq-find + (lambda (m) + (if (string-match "/\\(.*\\)/" m) + (let ((rx (match-string 1 m)) + (case-fold-search t)) + (string-match rx addr)) + (eq t (compare-strings addr nil nil m nil nil 'case-insensitive)))) + (mu4e-personal-addresses)))) + + +(defun mu4e-personal-or-alternative-address-p (addr) + "Is ADDR either a personal or an alternative address? + +That is, does it match either `mu4e-personal-address-p' or +`message-alternative-emails'. + +Note that this expanded definition of user-addresses is only used +in emacs, not in `mu' (e.g. when indexing). + +Also see `mu4e-personal-or-alternative-address-or-empty-p'." + (let ((alts message-alternative-emails)) + (or (mu4e-personal-address-p addr) + (cond + ((functionp alts) (funcall alts addr)) + ((stringp alts) (string-match alts addr)) + (t nil))))) + +(defun mu4e-personal-or-alternative-address-or-empty-p (addr) + "Is ADDR either a personal, alternative address or nil? + +This is like `mu4e-personal-or-alternative-address-p' but also +return t for _empty_ ADDR. This can be useful for use with +`message-dont-reply-to-names' since it can receive empty strings; +those can be filtered-out by returning t here. + +See #2680 for further details. " + (or (and addr (string= addr "")) + (mu4e-personal-or-alternative-address-p addr))) + + +;; Helpers + +;;; RFC2822 handling of phrases in mail-addresses +;; +;; The optional display-name contains a phrase, it sits before the +;; angle-addr as specified in RFC2822 for email-addresses in header +;; fields. Contributed by jhelberg. + +(defun mu4e--rfc822-phrase-type (ph) + "Return an atom or quoted-string for the phrase PH. +This checks for empty string first. Then quotes around the phrase +\(returning symbol `rfc822-quoted-string'). Then whether there is +a quote inside the phrase (returning symbol +`rfc822-containing-quote'). + +The reverse of the RFC atext definition is then tested. If it +matches, nil is returned, if not, it returns a symbol +`rfc822-atom'." + (cond + ((= (length ph) 0) 'rfc822-empty) + ((= (aref ph 0) ?\") + (if (string-match "\"\\([^\"\\\n]\\|\\\\.\\|\\\\\n\\)*\"" ph) + 'rfc822-quoted-string + 'rfc822-containing-quote)) ; starts with quote, but doesn't end with one + ((string-match-p "[\"]" ph) 'rfc822-containing-quote) + ((string-match-p "[\000-\037()\*<>@,;:\\\.]+" ph) nil) + (t 'rfc822-atom))) + +(defun mu4e--rfc822-quote-phrase (ph) + "Quote an RFC822 phrase PH only if necessary. +Atoms and quoted strings don't need quotes. The rest do. In +case a phrase contains a quote, it will be escaped." + (let ((type (mu4e--rfc822-phrase-type ph))) + (cond + ((eq type 'rfc822-atom) ph) + ((eq type 'rfc822-quoted-string) ph) + ((eq type 'rfc822-containing-quote) + (format "\"%s\"" + (replace-regexp-in-string "\"" "\\\\\"" ph))) + (t (format "\"%s\"" ph))))) + +(defsubst mu4e-contact-name (contact) + "Get the name of this CONTACT, or nil." + (plist-get contact :name)) + +(defsubst mu4e-contact-email (contact) + "Get the name of this CONTACT, or nil." + (plist-get contact :email)) + +(defsubst mu4e-contact-cons (contact) + "Convert a CONTACT plist into a old-style (name . email)." + (cons + (mu4e-contact-name contact) + (mu4e-contact-email contact))) + +(defsubst mu4e-contact-make (name email) + "Create a contact plist from NAME and EMAIL." + `(:name ,name :email ,email)) + +(defun mu4e-contact-full (contact) + "Get the full combination of name and email address from CONTACT." + (let* ((email (mu4e-contact-email contact)) + (name (mu4e-contact-name contact))) + (if (and name (> (length name) 0)) + (format "%s <%s>" (mu4e--rfc822-quote-phrase name) email) + email))) + + +(defun mu4e--update-contacts (contacts &optional tstamp) + "Receive a sorted list of CONTACTS newer than TSTAMP. +Update an internal set with it. + +This is used by the completion function in mu4e-compose." + (let ((n 0)) + (unless mu4e--contacts-set + (setq mu4e--contacts-set (make-hash-table :test 'equal :weakness nil + :size (length contacts)))) + (dolist (contact contacts) + (cl-incf n) + (when (functionp mu4e-contact-process-function) + (setq contact (funcall mu4e-contact-process-function contact))) + (when contact ;; note the explicit deccode; the strings we get are + ;; utf-8, but emacs doesn't know yet. + (puthash (decode-coding-string contact 'utf-8) t mu4e--contacts-set))) + (setq mu4e--contacts-tstamp (or tstamp "0")) + (unless (zerop n) + (mu4e-index-message "Contacts updated: %d; total %d" + n (hash-table-count mu4e--contacts-set))))) + +(defun mu4e-contacts-info () + "Display information about the contacts-cache. +For testing/debugging." + (interactive) + (with-current-buffer (get-buffer-create "*mu4e-contacts-info*") + (erase-buffer) + (insert (format "complete addresses: %s\n" + (if mu4e-compose-complete-addresses "yes" "no"))) + (insert (format "only personal addresses: %s\n" + (if mu4e-compose-complete-only-personal "yes" "no"))) + (insert (format "only addresses seen after: %s\n" + (or mu4e-compose-complete-only-after "no restrictions"))) + + (when mu4e--contacts-set + (insert (format "number of contacts cached: %d\n\n" + (hash-table-count mu4e--contacts-set))) + (maphash (lambda (contact _) + (insert (format "%s\n" contact))) mu4e--contacts-set)) + (pop-to-buffer "*mu4e-contacts-info*"))) + +(declare-function mu4e--server-contacts "mu4e-server") + +(defun mu4e--request-contacts-maybe () + "Maybe update the set of contacts for autocompletion. + +If `mu4e-compose-complete-addresses' is non-nil, get/update the +list of contacts we use for autocompletion; otherwise, do +nothing." + (when mu4e-compose-complete-addresses + (mu4e--server-contacts + mu4e-compose-complete-only-personal + mu4e-compose-complete-only-after + mu4e-compose-complete-max + mu4e--contacts-tstamp))) + +(provide 'mu4e-contacts) +;;; mu4e-contacts.el ends here diff --git a/mu4e/mu4e-context.el b/mu4e/mu4e-context.el new file mode 100644 index 0000000..98cfedc --- /dev/null +++ b/mu4e/mu4e-context.el @@ -0,0 +1,243 @@ +;;; mu4e-context.el --- Switching between settings -*- lexical-binding: t -*- + +;; Copyright (C) 2015-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; A mu4e 'context' is a set of variable-settings and functions, which can be +;; used e.g. to switch between accounts. + +;;; Code: + +(require 'mu4e-helpers) +(require 'mu4e-modeline) +(require 'mu4e-query-items) + + +;;; Configuration +(defcustom mu4e-context-policy 'ask-if-none + "The policy to determine the context when entering the mu4e main view. + +If the value is `always-ask', ask the user unconditionally. + +In all other cases, if any context matches (using its match +function), this context is used. Otherwise, if none of the +contexts match, we have the following choices: + +- `pick-first': pick the first of the contexts available (ie. the default) +- `ask': ask the user `ask-if-none': ask if there is no context yet, + otherwise leave it as it is +- nil: return nil; eaves the current context as is. + +Also see `mu4e-compose-context-policy'." + :type '(choice + (const :tag "Always ask what context to use, even if one matches" + always-ask) + (const :tag "Ask if none of the contexts match" ask) + (const :tag "Ask when there's no context yet" ask-if-none) + (const :tag "Pick the first context if none match" pick-first) + (const :tag "Don't change the context when none match" nil)) + :group 'mu4e) + + +(defvar mu4e-contexts nil + "The list of `mu4e-context' objects describing mu4e's contexts.") + +(defvar mu4e-context-changed-hook nil + "Hook run just *after* the context changed.") + +(defface mu4e-context-face + '((t :inherit mu4e-title-face :weight bold)) + "Face for displaying the context in the modeline." + :group 'mu4e-faces) + +(defvar mu4e--context-current nil + "The current context. +Internal; use `mu4e-context-switch' to change it.") + +(defun mu4e-context-current (&optional output) + "Get the currently active context, or nil if there is none. +When OUTPUT is non-nil, echo the name of the current context or +none." + (interactive "p") + (let ((ctx mu4e--context-current)) + (when output + (mu4e-message "Current context: %s" + (if ctx (mu4e-context-name ctx) "<none>"))) + ctx)) + +(cl-defstruct mu4e-context + "A mu4e context object with the following members: +- `name': the name of the context, eg. \"Work\" or \"Private\". +- `enter-func': a parameterless function invoked when entering + this context, or nil +- `leave-func':a parameterless function invoked when leaving this + context, or nil +- `match-func': a function called when composing a new message, + that takes a message plist for the message replied to or + forwarded, and nil otherwise. Before composing a new message, + `mu4e' switches to the first context for which `match-func' + returns t. +- `vars': variables to set when entering context." + name ;; name of the context, e.g. "work" + (enter-func nil) ;; function invoked when entering the context + (leave-func nil) ;; function invoked when leaving the context + (match-func nil) ;; function that takes a msg-proplist, and return t + ;; if it matches, nil otherwise + vars) ;; alist of variables. + + +(defun mu4e--context-ask-user (prompt) + "Let user choose some context based on its name with PROMPT." + (when mu4e-contexts + (let* ((names (seq-map (lambda (context) + (cons (mu4e-context-name context) context)) + mu4e-contexts)) + (context (mu4e-read-option prompt names))) + (or context (mu4e-error "No such context"))))) + +(defun mu4e-context-switch (&optional force name) + "Switch to a context with NAME. +Context must be part of `mu4e-contexts'; if NAME is nil, query user. + +If the new context is the same as the current context, only +switch (run associated functions) when prefix argument FORCE is +non-nil." + (interactive "P") + (unless mu4e-contexts + (mu4e-error "No contexts defined")) + (let* ((names (seq-map (lambda (context) + (cons (mu4e-context-name context) context)) + mu4e-contexts)) + (old-context mu4e--context-current) ; i.e., context before switch + (context + (if name + (cdr-safe (assoc name names)) + (mu4e--context-ask-user "Switch to context: ")))) + (unless context (mu4e-error "No such context")) + ;; if new context is same as old one, only switch with FORCE + (when (or force (not (eq context (mu4e-context-current)))) + (when (and (mu4e-context-current) + (mu4e-context-leave-func mu4e--context-current)) + (funcall (mu4e-context-leave-func mu4e--context-current))) + ;; enter the new context + (when (mu4e-context-enter-func context) + (funcall (mu4e-context-enter-func context))) + (when (mu4e-context-vars context) + (mapc (lambda (cell) + (set (car cell) (cdr cell))) + (mu4e-context-vars context))) + (setq mu4e--context-current context) + (run-hooks 'mu4e-context-changed-hook) + ;; refresh the cached query items if there was a context before; we have + ;; have different bookmarks/maildirs now. + (when old-context + (mu4e--query-items-refresh 'reset-baseline)) + (mu4e-message "Switched context to %s" + (mu4e-context-name context))) + context)) + +(defun mu4e--context-autoswitch (&optional msg policy) + "Automatically switch to some context. + +When contexts are defined but there is no context yet, switch to +the first whose :match-func return non-nil. If none of them +match, return the first. For MSG and POLICY, see +`mu4e-context-determine'." + (when mu4e-contexts + (let ((context (mu4e-context-determine msg policy))) + (when context (mu4e-context-switch + nil (mu4e-context-name context)))))) + +(defun mu4e-context-determine (msg &optional policy) + "Return the first context where match-func evaluate to non-nil. + +MSG points to the plist for the message replied to or forwarded, +or nil if there is no such MSG; similar to what +`mu4e-compose-pre-hook' does. + +POLICY specifies how to do the determination. If POLICY is +`always-ask', we ask the user unconditionally. + +In all other cases, if any context matches (using its match +function), this context is returned. If none of the contexts +match, POLICY determines what to do: + +- `pick-first': pick the first of the contexts available +- `ask': ask the user +- `ask-if-none': ask if there is no context yet +- otherwise, return nil. Effectively, this leaves the current context +as it is." + (when mu4e-contexts + (if (eq policy 'always-ask) + (mu4e--context-ask-user "Select context: ") + (or ;; is there a matching one? + (seq-find (lambda (context) + (when (mu4e-context-match-func context) + (funcall (mu4e-context-match-func context) msg))) + mu4e-contexts) + ;; no context found yet; consult policy + (pcase policy + ('pick-first (car mu4e-contexts)) + ('ask (mu4e--context-ask-user "Select context: ")) + ('ask-if-none (or (mu4e-context-current) + (mu4e--context-ask-user "Select context: "))) + (_ nil)))))) + +(defmacro with-mu4e-context-vars (context &rest body) + "Evaluate BODY, with variables let-bound for CONTEXT (if any). +`funcall'." + (declare (indent 2)) + `(let* ((vars (and ,context (mu4e-context-vars ,context)))) + (cl-progv ;; XXX: perhaps use eval's lexical environment instead of progv? + (mapcar (lambda(cell) (car cell)) vars) + (mapcar (lambda(cell) (cdr cell)) vars) + (eval ,@body)))) + +(defun mu4e--context-modeline-item () + "Propertized string with the current context or nil." + (when-let* ((ctx (mu4e-context-current)) + (name (and ctx (mu4e-context-name ctx)))) + (concat + "<" + (propertize + name + 'face 'mu4e-context-face + 'help-echo (format "mu4e context: %s" name)) + ">"))) + +(define-minor-mode mu4e-context-minor-mode + "Mode for switching the mu4e context." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + (mu4e--modeline-register #'mu4e--context-modeline-item)) + +(defvar mu4e--context-menu-items + '("--" + ["Switch-context" mu4e-context-switch + :help "Switch the mu4e context"]) + "Easy menu items for mu4e-context.") + +;;; +(provide 'mu4e-context) +;;; mu4e-context.el ends here diff --git a/mu4e/mu4e-contrib.el b/mu4e/mu4e-contrib.el new file mode 100644 index 0000000..1da3c9b --- /dev/null +++ b/mu4e/mu4e-contrib.el @@ -0,0 +1,201 @@ +;;; mu4e-contrib.el --- User-contributed functions -*- lexical-binding: t -*- + +;; Copyright (C) 2013-2023 Dirk-Jan C. Binnema + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Some user-contributed functions for mu4e + +;;; Code: + +(require 'mu4e-headers) +(require 'mu4e-view) +(require 'bookmark) +(require 'eshell) + + +;;; Various simple commands +(defun mu4e-headers-mark-all-unread-read () + "Put a ! \(read) mark on all visible unread messages." + (interactive) + (mu4e-headers-mark-for-each-if + (cons 'read nil) + (lambda (msg _param) + (memq 'unread (mu4e-msg-field msg :flags))))) + +(defun mu4e-headers-flag-all-read () + "Flag all visible messages as \"read\"." + (interactive) + (mu4e-headers-mark-all-unread-read) + (mu4e-mark-execute-all t)) + +(defun mu4e-headers-mark-all () + "Mark all headers for some action. +Ask user what action to execute." + (interactive) + (mu4e-headers-mark-for-each-if + (cons 'something nil) + (lambda (_msg _param) t)) + (mu4e-mark-execute-all)) + + + +;;; Bogofilter/SpamAssassin +;; +;; Support for handling spam with Bogofilter with the possibility +;; to define it for SpamAssassin, contributed by Gour. +;; +;; To add the actions to the menu, you can use something like: +;; +;; (add-to-list 'mu4e-headers-actions +;; '("sMark as spam" . mu4e-register-msg-as-spam) t) +;; (add-to-list 'mu4e-headers-actions +;; '("hMark as ham" . mu4e-register-msg-as-ham) t) + +(defvar mu4e-register-as-spam-cmd nil + "Command for invoking spam processor to register message as spam. +For example for bogofilter, use \"/usr/bin/bogofilter -Ns < %s\"") + +(defvar mu4e-register-as-ham-cmd nil + "Command for invoking spam processor to register message as ham. +For example for bogofile, use \"/usr/bin/bogofilter -Sn < %s\"") + +(defun mu4e-register-msg-as-spam (msg) + "Register MSG as spam." + (interactive) + (let* ((path (shell-quote-argument (mu4e-message-field msg :path))) + (command (format mu4e-register-as-spam-cmd path))) + (shell-command command)) + (mu4e-mark-at-point 'delete nil)) + +(defun mu4e-register-msg-as-ham (msg) + "Register MSG as ham." + (interactive) + (let* ((path (shell-quote-argument(mu4e-message-field msg :path))) + (command (format mu4e-register-as-ham-cmd path))) + (shell-command command)) + (mu4e-mark-at-point 'something nil)) + +;; (add-to-list 'mu4e-view-actions +;; '("sMark as spam" . mu4e-view-register-msg-as-spam) t) +;; (add-to-list 'mu4e-view-actions +;; '("hMark as ham" . mu4e-view-register-msg-as-ham) t) + +(defun mu4e-view-register-msg-as-spam (msg) + "Register MSG as spam (view mode)." + (interactive) + (let* ((path (shell-quote-argument (mu4e-message-field msg :path))) + (command (format mu4e-register-as-spam-cmd path))) + (shell-command command)) + (mu4e-view-mark-for-delete)) + +(defun mu4e-view-register-msg-as-ham (msg) + "Mark MSG as ham (view mode)." + (interactive) + (let* ((path (shell-quote-argument(mu4e-message-field msg :path))) + (command (format mu4e-register-as-ham-cmd path))) + (shell-command command)) + (mu4e-view-mark-for-something)) + + +;;; Eshell functions +;; +;; Code for `gnus-dired-attached' modified to run from eshell, +;; allowing files to be attached to an email via mu4e using the +;; eshell. Does not depend on gnus. + + +(defun mu4e--active-composition-buffers () + "Return all active mu4e composition buffers." + (let (buffers) + (save-excursion + (dolist (buffer (buffer-list t)) + (set-buffer buffer) + (when (eq major-mode 'mu4e-compose-mode) + (push (buffer-name buffer) buffers)))) + (nreverse buffers))) + + + +;; backward compat until 27.1 is univeral. +(defalias 'mu4e--flatten-list + (if (fboundp 'flatten-list) + #'flatten-list + (with-no-warnings + #'eshell-flatten-list))) + +;; backward compat ntil 28.1 is universal. +(defalias 'mu4e--mm-default-file-type + (if (fboundp 'mm-default-file-type) + #'mm-default-file-type + (with-no-warnings + #'mm-default-file-encoding))) + +(defun eshell/mu4e-attach (&rest args) + "Attach files to a mu4e message using eshell with ARGS. +If no mu4e buffers found, compose a new message and then attach +the file." + (let ((destination nil) + (files-str nil) + (bufs nil) + ;; Remove directories from the list + (files-to-attach + (delq nil (mapcar + (lambda (f) (if (or (not (file-exists-p f)) + (file-directory-p f)) + nil + (expand-file-name f))) + (mu4e--flatten-list (reverse args)))))) + ;; warn if user tries to attach without any files marked + (if (null files-to-attach) + (error "No files to attach") + (setq files-str + (mapconcat + (lambda (f) (file-name-nondirectory f)) + files-to-attach ", ")) + (setq bufs (mu4e--active-composition-buffers)) + ;; set up destination mail composition buffer + (if (and bufs + (y-or-n-p "Attach files to existing mail composition buffer? ")) + (setq destination + (if (= (length bufs) 1) + (get-buffer (car bufs)) + (let ((prompt (mu4e-format "%s" "Attach to buffer"))) + (substring-no-properties + (funcall mu4e-completing-read-function prompt + bufs))))) + ;; setup a new mail composition buffer + (if (y-or-n-p "Compose new mail and attach this file? ") + (progn (mu4e-compose-new) + (setq destination (current-buffer))))) + ;; if buffer was found, set buffer to destination buffer, and attach files + (if (not (eq destination 'nil)) + (progn (set-buffer destination) + (goto-char (point-max)) ; attach at end of buffer + (while files-to-attach + (mml-attach-file (car files-to-attach) + (or (mu4e--mm-default-file-type + (car files-to-attach)) + "application/octet-stream") nil) + (setq files-to-attach (cdr files-to-attach))) + (message "Attached file(s) %s" files-str)) + (message "No buffer to attach file to."))))) + +;;; _ +(provide 'mu4e-contrib) +;;; mu4e-contrib.el ends here diff --git a/mu4e/mu4e-draft.el b/mu4e/mu4e-draft.el new file mode 100644 index 0000000..3b94cdd --- /dev/null +++ b/mu4e/mu4e-draft.el @@ -0,0 +1,746 @@ +;;; mu4e-draft.el --- Helpers for m4e-compose -*- lexical-binding: t -*- + +;; Copyright (C) 2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Implements various helper functions for mu4e-compose. This all +;; look a little convoluted since we need to subvert the gnus/message +;; functions a bit to work with mu4e. + +(require 'message) +(require 'mu4e-config) +(require 'mu4e-helpers) +(require 'mu4e-contacts) +(require 'mu4e-folders) +(require 'mu4e-message) +(require 'mu4e-context) +(require 'mu4e-window) + +;;; Code: + +(declare-function mu4e-compose-mode "mu4e-compose") +(declare-function mu4e "mu4e") + +(defcustom mu4e-compose-crypto-policy + '(encrypt-encrypted-replies sign-encrypted-replies) + "Policy to control when messages will be signed/encrypted. + +The value is a list which influence the way draft messages are +created. Specifically, it might contain: + +- `sign-all-messages': Always add a signature. +- `sign-new-messages': Add a signature to new message, ie. + messages that aren't responses to another message. +- `sign-forwarded-messages': Add a signature when forwarding + a message +- `sign-edited-messages': Add a signature to drafts +- `sign-all-replies': Add a signature when responding to + another message. +- `sign-plain-replies': Add a signature when responding to + non-encrypted messages. +- `sign-encrypted-replies': Add a signature when responding + to encrypted messages. + +It should be noted that certain symbols have priorities over one +another. So `sign-all-messages' implies `sign-all-replies', which +in turn implies `sign-plain-replies'. Adding both to the set, is +not a contradiction, but a redundant configuration. + +All `sign-*' options have a `encrypt-*' analogue." + :type '(set :greedy t + (const :tag "Sign all messages" sign-all-messages) + (const :tag "Encrypt all messages" encrypt-all-messages) + (const :tag "Sign new messages" sign-new-messages) + (const :tag "Encrypt new messages" encrypt-new-messages) + (const :tag "Sign forwarded messages" sign-forwarded-messages) + (const :tag "Encrypt forwarded messages" + encrypt-forwarded-messages) + (const :tag "Sign edited messages" sign-edited-messages) + (const :tag "Encrypt edited messages" edited-forwarded-messages) + (const :tag "Sign all replies" sign-all-replies) + (const :tag "Encrypt all replies" encrypt-all-replies) + (const :tag "Sign replies to plain messages" sign-plain-replies) + (const :tag "Encrypt replies to plain messages" + encrypt-plain-replies) + (const :tag "Sign replies to encrypted messages" + sign-encrypted-replies) + (const :tag "Encrypt replies to encrypted messages" + encrypt-encrypted-replies)) + :group 'mu4e-compose) + +;;; Crypto +(defun mu4e--prepare-crypto (parent compose-type) + "Possibly encrypt or sign a message based on PARENT and COMPOSE-TYPE. +See `mu4e-compose-crypto-policy' for more details." + (let* ((encrypted-p + (and parent (memq 'encrypted (mu4e-message-field parent :flags)))) + (encrypt + (or (memq 'encrypt-all-messages mu4e-compose-crypto-policy) + (and (memq 'encrypt-new-messages mu4e-compose-crypto-policy) + (eq compose-type 'new)) ;; new messages + (and (eq compose-type 'forward) ;; forwarded + (memq 'encrypt-forwarded-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'edit) ;; edit + (memq 'encrypt-edited-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) ;; all replies + (memq 'encrypt-all-replies mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) (not encrypted-p) ;; plain replies + (memq 'encrypt-plain-replies mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) encrypted-p + (memq 'encrypt-encrypted-replies + mu4e-compose-crypto-policy)))) ;; encrypted replies + (sign + (or (memq 'sign-all-messages mu4e-compose-crypto-policy) + (and (eq compose-type 'new) ;; new messages + (memq 'sign-new-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'forward) ;; forwarded messages + (memq 'sign-forwarded-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'edit) ;; edited messages + (memq 'sign-edited-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) ;; all replies + (memq 'sign-all-replies mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) (not encrypted-p) ;; plain replies + (memq 'sign-plain-replies mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) encrypted-p ;; encrypted replies + (memq 'sign-encrypted-replies mu4e-compose-crypto-policy))))) + (cond ((and sign encrypt) (mml-secure-message-sign-encrypt)) + (sign (mml-secure-message-sign)) + (encrypt (mml-secure-message-encrypt))))) + +(defcustom mu4e-sent-messages-behavior 'sent + "Determines what mu4e does with sent messages. + +This is one of the symbols: +* `sent' move the sent message to the Sent-folder (`mu4e-sent-folder') +* `trash' move the sent message to the Trash-folder (`mu4e-trash-folder') +* `delete' delete the sent message. + +Note, when using GMail/IMAP, you should set this to either +`trash' or `delete', since GMail already takes care of keeping +copies in the sent folder. + +Alternatively, `mu4e-sent-messages-behavior' can be a function +which takes no arguments, and which should return one of the mentioned +symbols, for example: + + (setq mu4e-sent-messages-behavior (lambda () + (if (string= (message-sendmail-envelope-from) \"foo@example.com\") + \\='delete \\='sent))) + +The various `message-' functions from `message-mode' are available +for querying the message information." + :type '(choice (const :tag "move message to mu4e-sent-folder" sent) + (const :tag "move message to mu4e-trash-folder" trash) + (const :tag "delete message" delete)) + :group 'mu4e-compose) + +(defcustom mu4e-compose-context-policy 'ask + "Policy for determining the context when composing a new message. + +If the value is `always-ask', ask the user unconditionally. + +In all other cases, if any context matches (using its match +function), this context is used. Otherwise, if none of the +contexts match, we have the following choices: + +- `pick-first': pick the first of the contexts available (ie. the default) +- `ask': ask the user +- `ask-if-none': ask if there is no context yet, otherwise leave it as it is +- nil: return nil; leaves the current context as is. + +Also see `mu4e-context-policy'." + :type '(choice + (const :tag "Always ask what context to use" always-ask) + (const :tag "Ask if none of the contexts match" ask) + (const :tag "Ask when there's no context yet" ask-if-none) + (const :tag "Pick the first context if none match" pick-first) + (const :tag "Don't change the context when none match" nil)) + :safe 'symbolp + :group 'mu4e-compose) + +;; +;; display the ready-to-go display buffer in the desired way. +;; +(defun mu4e--display-draft-buffer (cbuf) + "Display the message composition buffer CBUF. +Display is influenced by `mu4e-compose-switch'." + (let ((func + (pcase mu4e-compose-switch + ('nil #'switch-to-buffer) + ('window #'switch-to-buffer-other-window) + ((or 'frame 't) #'switch-to-buffer-other-frame) + ('display-buffer #'display-buffer) + (_ (mu4e-error "Invalid mu4e-compose-switch"))))) + (funcall func cbuf))) + +(defvar mu4e-user-agent-string + (format "mu4e %s; emacs %s" mu4e-mu-version emacs-version) + "The User-Agent string for mu4e, or nil.") + +;;; Runtime variables; useful for user-hooks etc. +;; mu4e-compose-parent-message & mu4e-compose-type are buffer-local and +;; permanent-local so they'll survive the mode change to mu4e-compose-mode and +;; we can use them in the corresponding mode-hook. +(defvar-local mu4e-compose-parent-message nil + "The parent message plist. +This is the message being replied to, forwarded or edited; used +in `mu4e-compose-pre-hook'. For new (non-reply, forward etc.) +messages, it is nil.") +(put 'mu4e-compose-parent-message 'permanent-local t) + +(defvar-local mu4e-compose-type nil + "The compose-type for the current message.") +(put 'mu4e-compose-type 'permanent-local t) + +;;; Filenames +(defun mu4e--draft-basename() + "Construct a randomized filename for a message with flags FLAGSTR. +It looks something like + <time>-<random>.<hostname> + +This filename is used for the draft message and the sent message, +depending on `mu4e-sent-messages-behavior'." + (let* ((sysname (if (fboundp 'system-name) + (system-name) (with-no-warnings system-name))) + (sysname (if (string= sysname "") "localhost" sysname)) + (hostname (downcase + (save-match-data + (substring sysname + (string-match "^[^.]+" sysname) + (match-end 0)))))) + (format "%s.%04x%04x%04x%04x.%s" + (format-time-string "%s" (current-time)) + (random 65535) (random 65535) (random 65535) (random 65535) + hostname))) + +(defun mu4e--draft-message-path (base-name &optional parent) + "Construct a draft message path, based on PARENT if provided. + +PARENT is either nil or the original message (being replied + to/forwarded etc.), and is used to determine the draft folder. +BASE-NAME is the base filename without any Maildir decoration." + (let ((draft-dir (mu4e-get-drafts-folder parent))) + (mu4e-join-paths + (mu4e-root-maildir) draft-dir "cur" + (format "%s%s2,DS" base-name mu4e-maildir-info-delimiter)))) + +(defun mu4e--fcc-path (base-name &optional parent) + "Construct a Fcc: path, based on PARENT and `mu4e-sent-messages-behavior'. + +PARENT is either nil or the original message (being replied +to/forwarded etc.), and is used to determine the sent folder, +together with `mu4e-sent-messages-behavior'. BASE-NAME is the +base filename without any Maildir decoration. + +Returns the path for the sent message, either in the sent or +trash folder, or nil if the message should be removed after +sending." + (let* ((behavior + (if (and (functionp mu4e-sent-messages-behavior) + ;; don't interpret 'delete as a function... + (not (eq mu4e-sent-messages-behavior 'delete))) + (funcall mu4e-sent-messages-behavior) + mu4e-sent-messages-behavior)) + (sent-dir + (pcase behavior + ('delete nil) + ('trash (mu4e-get-trash-folder parent)) + ('sent (mu4e-get-sent-folder parent)) + (_ (mu4e-error "Error in `mu4e-sent-messages-behavior'"))))) + (when sent-dir + (mu4e-join-paths + (mu4e-root-maildir) sent-dir "cur" + (format "%s%s2,S" base-name mu4e-maildir-info-delimiter))))) + + +(defconst mu4e--header-separator + ;; XX properties don't show... why not? + (propertize "--text follows this line--" 'read-only t 'intangible t) + "Line used to separate headers from text in messages being composed.") + +(defun mu4e--delimit-headers (&optional undelimit) + "Delimit or undelimit (with UNDELIMIT) headers." + (let ((mail-header-separator (substring-no-properties mu4e--header-separator)) + (inhibit-read-only t)) + (save-excursion (mail-sendmail-undelimit-header)) ;; clear first + (unless undelimit (save-excursion (mail-sendmail-delimit-header))))) + +(defun mu4e--decoded-message (msg &optional headers-only) + "Get the message MSG, decoded as a string. +With HEADERS-ONLY non-nil, only include the headers part." + (with-temp-buffer + (setq-local gnus-article-decode-hook + '(article-decode-charset + article-decode-encoded-words + article-decode-idna-rhs + article-treat-non-ascii + article-remove-cr + article-de-base64-unreadable + article-de-quoted-unreadable) + gnus-inhibit-mime-unbuttonizing nil + gnus-unbuttonized-mime-types '(".*/.*") + gnus-original-article-buffer (current-buffer)) + (insert-file-contents-literally + (mu4e-message-readable-path msg) nil nil nil t) + ;; remove the body / attachments and what not. + (when headers-only + (rfc822-goto-eoh) + (delete-region (point) (point-max))) + ;; in rare (broken) case, if a message-id is missing use the generated one + ;; from mu. + (mu4e--delimit-headers) + (unless (message-field-value "Message-Id") + (goto-char (point-min)) + (insert (format "Message-Id: <%s>\n" (plist-get msg :message-id)))) + (mu4e--delimit-headers 'undelimit) + (ignore-errors (run-hooks 'gnus-article-decode-hook)) + (buffer-substring-no-properties (point-min) (point-max)))) + +(defvar mu4e--draft-buffer-max-name-length 48) +(defun mu4e--draft-set-friendly-buffer-name () + "Use some friendly name for this draft buffer." + (let* ((subj (message-field-value "subject")) + (subj (if (or (not subj) (string-match "^[:blank:]*$" subj)) + "No subject" subj))) + (rename-buffer (generate-new-buffer-name + (format "\"%s\"" + (truncate-string-to-width subj + mu4e--draft-buffer-max-name-length + 0 nil t))) + (buffer-name)))) + +;; hook impls + +(defun mu4e--fcc-handler (msgpath) + "Handle Fcc: for MSGPATH. +This ensures that a copy of a sent messages ends up in the +appropriate sent-messages folder. If MSGPATH is nil, do nothing." + (when msgpath + (let* ((target-dir (file-name-directory msgpath)) + (target-mdir (file-name-directory target-dir))) + ;; create maildir if needed + (unless (file-exists-p target-mdir) + (make-directory + (mu4e-join-paths target-mdir "cur" 'parents)) + (make-directory + (mu4e-join-paths target-mdir "new" 'parents))) + (write-file msgpath) + (mu4e--server-add msgpath)))) + +;; save / send hooks + +(defvar-local mu4e--compose-undo nil + "Remember the undo-state.") + +(defun mu4e--compose-before-save () + "Function called just before the draft buffer is saved." + ;; This does 3 things: + ;; - set the Message-Id if not already + ;; - set the Date if not already + ;; - (temporarily) remove the mail-header separator + (setq mu4e--compose-undo buffer-undo-list) + (save-excursion + (unless (message-field-value "Message-ID") + (message-generate-headers '(Message-ID))) + ;; older Emacsen (<= 28 perhaps?) won't update the Date + ;; if there already is one; so make sure it's gone. + (message-remove-header "Date") + (message-generate-headers '(Date Subject From)) + (mu4e--delimit-headers 'undelimit))) ;; remove separator + +(defun mu4e--set-parent-flags (path) + "Set flags for replied-to and forwarded for the message at PATH. +That is, set the `replied' \"R\" flag on messages we replied to, +and the `passed' \"F\" flag on message we have forwarded. + +If a message has an \"In-Reply-To\" header, it is considered a +reply to the message with the corresponding message id. +Otherwise, if it does not have an \"In-Reply-To\" header, but +does have a \"References:\" header, it is considered to be a +forward message for the message corresponding with the /last/ +message-id in the references header. + +If the message has been determined to be either a forwarded +message or a reply, we instruct the server to update that message +with resp. the \"P\" (passed) flag for a forwarded message, or +the \"R\" flag for a replied message. The original messages are +also marked as Seen. + +Function assumes that it is executed in the context of the +message buffer." + (when-let ((buf (find-file-noselect path))) + (with-current-buffer buf + (let ((in-reply-to (message-field-value "in-reply-to")) + (forwarded-from) + (references (message-field-value "references"))) + (unless in-reply-to + (when references + (with-temp-buffer ;; inspired by `message-shorten-references'. + (insert references) + (goto-char (point-min)) + (let ((refs)) + (while (re-search-forward "<[^ <]+@[^ <]+>" nil t) + (push (match-string 0) refs)) + ;; the last will be the first + (setq forwarded-from (car refs)))))) + ;; remove the <> and update the flags on the server-side. + (when (and in-reply-to (string-match "<\\(.*\\)>" in-reply-to)) + (mu4e--server-move (match-string 1 in-reply-to) nil "+R-N")) + (when (and forwarded-from (string-match "<\\(.*\\)>" forwarded-from)) + (mu4e--server-move (match-string 1 forwarded-from) nil "+P-N")))))) + +(defun mu4e--compose-after-save() + "Function called immediately after the draft buffer is saved." + ;; This does 3 things: + ;; - restore the mail-header-separator (see mu4e--compose-before-save) + ;; - update the buffer name (based on the message subject + ;; - tell the mu server about the updated draft message + (mu4e--delimit-headers) + (mu4e--draft-set-friendly-buffer-name) + ;; tell the server + (mu4e--server-add (buffer-file-name)) + ;; restore history. + (set-buffer-modified-p nil) + (setq buffer-undo-list mu4e--compose-undo)) + +(defun mu4e-sent-handler (docid path) + "Handler called with DOCID and PATH for the just-sent message. +For Forwarded ('Passed') and Replied messages, try to set the +appropriate flag at the message forwarded or replied-to." + ;; XXX we don't need this function anymore here, but we have an external + ;; caller in mu4e-icalendar... we should update that. + (mu4e--set-parent-flags path) + ;; if the draft file exists, remove it now. + (when (file-exists-p path) + (mu4e--server-remove docid))) + +(defun mu4e--send-harden-newlines () + "Set the hard property to all newlines." + (save-excursion + (goto-char (point-min)) + (while (search-forward "\n" nil t) + (put-text-property (1- (point)) (point) 'hard t)))) + +(defun mu4e--compose-before-send () + "Function called just before sending a message." + ;; Remove References: if In-Reply-To: is missing. + ;; This allows the user to effectively start a new message-thread by + ;; removing the In-Reply-To header. + (when (eq mu4e-compose-type 'reply) + (unless (message-field-value "In-Reply-To") + (message-remove-header "References"))) + (when use-hard-newlines + (mu4e--send-harden-newlines)) + ;; now handle what happens _after_ sending; typically, draft is gone and + ;; the sent message appears in sent. Update flags for related messages, + ;; i.e. for Forwarded ('Passed') and Replied messages, try to set the + ;; appropriate flag at the message forwarded or replied-to. + (add-hook 'message-sent-hook + (lambda () + (when-let ((fcc-path (message-field-value "Fcc"))) + (mu4e--set-parent-flags fcc-path) + ;; we end up with a ((buried) buffer here, visiting the + ;; fcc-path; not quite sure why. But let's get rid of it (#2681) + (when-let ((buf (find-buffer-visiting fcc-path))) + (kill-buffer buf)) + ;; remove draft + (when-let ((draft (buffer-file-name))) + (mu4e--server-remove draft)))) + nil t)) + +;; overrides for message-* functions +;; +;; mostly some magic because the message-reply/-forward/... functions want to +;; create and switch to buffer by themselves; but mu4e wants to control +;; when/where the buffers are shown so we subvert the message-functions and get +;; the buffer without display it. + +(defvar mu4e--message-buf nil + "The message buffer created by (overridden) message-* functions.") + +(defun mu4e--message-pop-to-buffer (name &optional _switch) + "Mu4e override for `message-pop-to-buffer'. +Creates a buffer NAME and returns it." + (set-buffer (get-buffer-create name)) + (erase-buffer) + (setq mu4e--message-buf (current-buffer))) + +(defun mu4e--message-is-yours-p () + "Mu4e's override for `message-is-yours-p'." + (seq-some (lambda (field) + (if-let ((recip (message-field-value field))) + (mu4e-personal-or-alternative-address-p + (car (mail-header-parse-address recip))))) + '("From" "Sender"))) + +(defmacro mu4e--validate-hidden-buffer (&rest body) + "Macro to evaluate BODY and asserts that it yields a valid buffer. +Where valid means that it is a live an non-active buffer. +Returns said buffer." + `(let ((buf (progn ,@body))) + (cl-assert (buffer-live-p buf)) + (cl-assert (not (eq buf (window-buffer (selected-window))))) + buf)) + +(defun mu4e--message-call (func &rest params) + "Call message/gnus functions from a mu4e-context. +E.g., functions such as `message-reply' or `message-forward', but +manipulate such that they do *not* switch to the created buffer, +but merely return it. + +FUNC is the function to call and PARAMS are its parameters. + +For replying/forwarding, this functions expects to be called +while in a buffer with the to-be-forwarded/replied-to message." + (let* ((message-this-is-mail t) + (message-generate-headers-first nil) + (message-newsreader mu4e-user-agent-string) + (message-mail-user-agent nil)) + (cl-letf + ;; `message-pop-to-buffer' attempts switching the visible buffer; + ;; instead, we manipulate it to _return_ the buffer. + (((symbol-function #'message-pop-to-buffer) + #'mu4e--message-pop-to-buffer) + ;; teach `message-is-yours-p' about how mu4e defines that + ((symbol-function #'message-is-yours-p) + #'mu4e--message-is-yours-p)) + ;; also turn off all the gnus crypto handling, we do that ourselves.. + (setq-local gnus-message-replysign nil + gnus-message-replyencrypt nil + gnus-message-replysignencrypted nil) + (setq mu4e--message-buf nil) + (apply func params)) + (mu4e--validate-hidden-buffer mu4e--message-buf))) +;; +;; make the draft buffer ready for use. +;; + +(defun mu4e--jump-to-a-reasonable-place () + "Jump to a reasonable place for writing an email." + (if (not (message-field-value "To")) + (message-goto-to) + (if (not (message-field-value "Subject")) + (message-goto-subject) + (pcase message-cite-reply-position + ((or 'above 'traditional) (message-goto-body)) + (_ (when (message-goto-signature) (forward-line -2))))))) + +(defvar mu4e-draft-hidden-headers + (append message-hidden-headers '("^User-agent:" "^Fcc:")) + "Message headers to hide when composing. +This is mu4e's version of `message-hidden-headers'.") + +(defun mu4e--prepare-draft (&optional parent) + "Get ready for message composition. +PARENT is the parent message, if any." + (unless (mu4e-running-p) (mu4e 'background)) ;; start if needed + (mu4e--context-autoswitch parent mu4e-compose-context-policy)) + +(defun mu4e--prepare-draft-headers (compose-type) + "Add extra headers for message based on COMPOSE-TYPE." + (message-generate-headers + (seq-filter #'identity ;; ensure needed headers are generated. + `(From Subject Date Message-ID + ,(when (memq compose-type '(reply forward)) 'References) + ,(when (eq compose-type 'reply) 'In-Reply-To) + ,(when message-newsreader 'User-Agent) + ,(when message-user-organization 'Organization))))) + +(defun mu4e--prepare-draft-buffer (compose-type parent) + "Prepare the current buffer as a draft-buffer. +COMPOSE-TYPE and PARENT are as in `mu4e--draft'." + (cl-assert (member compose-type '(reply forward edit new))) + (cl-assert (eq (if parent t nil) + (if (member compose-type '(reply forward)) t nil))) + ;; remember some variables, e.g for user hooks. These are permanent-local + ;; hence survive the mode-switch below (we do this so these useful vars are + ;; available in mode-hooks. + (setq-local + mu4e-compose-parent-message parent + mu4e-compose-type compose-type) + + ;; draft path + (unless (eq compose-type 'edit) + (set-visited-file-name ;; make it a draft file + (mu4e--draft-message-path (mu4e--draft-basename) parent))) + ;; fcc + (when-let ((fcc-path (mu4e--fcc-path (mu4e--draft-basename) parent))) + (message-add-header (concat "Fcc: " fcc-path "\n"))) + + (mu4e--prepare-draft-headers compose-type) + (mu4e--prepare-crypto parent compose-type) + ;; set the attachment dir to something more reasonable than the draft + ;; directory. + (setq default-directory (mu4e-determine-attachment-dir)) + (mu4e--draft-set-friendly-buffer-name) + + ;; now, switch to compose mode + (mu4e-compose-mode) + + ;; hide some internal headers + (let ((message-hidden-headers mu4e-draft-hidden-headers)) + (message-hide-headers)) + + ;; hooks + (add-hook 'before-save-hook #'mu4e--compose-before-save nil t) + (add-hook 'after-save-hook #'mu4e--compose-after-save nil t) + (add-hook 'message-send-hook #'mu4e--compose-before-send nil t) + (setq-local message-fcc-handler-function #'mu4e--fcc-handler) + + (mu4e--jump-to-a-reasonable-place) + + (set-buffer-modified-p nil) + (undo-boundary)) + +;; +;; mu4e-compose-pos-hook helpers + +(defvar mu4e--before-draft-window-config nil + "The window configuration just before creating the draft.") + +(defun mu4e-compose-post-restore-window-configuration() + "Function that perhaps restores the window configuration. +I.e. the configuration just before the draft buffer appeared. +This is for use in `mu4e-compose-post-hook'. +See `set-window-configuration' for further details." + (when mu4e--before-draft-window-config + ;;(message "RESTORE to %s" mu4e--before-draft-window-config) + (set-window-configuration mu4e--before-draft-window-config) + (setq mu4e--before-draft-window-config nil))) + +(defvar mu4e--draft-activation-frame nil + "Frame from which composition was activated. +Used internally for mu4e-compose-post-kill-frame.") + +(defun mu4e-compose-post-kill-frame () + "Function that perhaps kills the composition frame. +This is for use in `mu4e-compose-post-hook'." + (let ((msgframe (selected-frame))) + ;;(message "kill frame? %s %s" mu4e--draft-activation-frame msgframe) + (when (and (frame-live-p msgframe) + (not (eq mu4e--draft-activation-frame msgframe))) + (delete-frame msgframe)))) + +(defvar mu4e-message-post-action nil + "Runtime variable for use with `mu4e-compose-post-hook'. +It contains a symbol denoting the action that triggered the hook, +either `send', `exit', `kill' or `postpone'.") + +(defvar mu4e-compose-post-hook) +(defun mu4e--message-post-actions (trigger) + "Invoked after we're done with a message. + +I.e. this multiplexes the `message-(send|exit|kill|postpone)-actions'; +with the mu4e-message-post-action set accordingly." + (setq mu4e-message-post-action trigger) + (run-hooks 'mu4e-compose-post-hook)) + +(defun mu4e--prepare-post (&optional oldframe oldwindconf) + "Prepare the `mu4e-compose-post-hook` handling. + +Set up some message actions. In particular, handle closing frames +when we created it. OLDFRAME is the frame from which the +message-composition was triggered. OLDWINDCONF is the current +window configuration." + ;; remember current frame & window conf + (setq mu4e--draft-activation-frame oldframe + mu4e--before-draft-window-config oldwindconf) + + ;; make message's "post" hooks local, and multiplex them + (make-local-variable 'message-send-actions) + (make-local-variable 'message-postpone-actions) + (make-local-variable 'message-exit-actions) + (make-local-variable 'message-kill-actions) + + (push (lambda () (mu4e--message-post-actions 'send)) + message-send-actions) + (push (lambda () (mu4e--message-post-actions 'postpone)) + message-postpone-actions) + (push (lambda () (mu4e--message-post-actions 'exit)) + message-exit-actions) + (push (lambda () (mu4e--message-post-actions 'kill)) + message-kill-actions)) + +;; +;; creating drafts +;; + +(defun mu4e--draft (compose-type compose-func &optional parent) + "Create a new message draft. + +This is the central access point for creating new mail buffers; +when there's a parent message, use `mu4e--compose-with-parent'. + +COMPOSE-TYPE is the type of message to create. COMPOSE-FUNC is a +function that must return a buffer that satisfies +`mu4e--validate-hidden-buffer'. + +Optionally, PARENT is the message parent or nil. For compose-type +`reply' and `forward' we require a PARENT; for the other compose +it must be nil. + +After this, user is presented with a message composition buffer. + +Returns the new buffer." + (mu4e--prepare-draft parent) + ;; evaluate BODY; this must yield a hidden, live buffer. This is evaluated in + ;; a temp buffer with contains the parent-message, if any. if there's a + ;; PARENT, load the corresponding message into a temp-buffer before calling + ;; compose-func + (let ((draft-buffer) + (oldframe (selected-frame)) + (oldwinconf (current-window-configuration))) + (with-temp-buffer + ;; provide a temp buffer so the compose-func can do its thing + (setq draft-buffer (mu4e--validate-hidden-buffer (funcall compose-func))) + (with-current-buffer draft-buffer + ;; we have our basic buffer; turn it into a full mu4e composition + ;; buffer. + (mu4e--prepare-draft-buffer compose-type parent))) + ;; we're ready for composition; let's display it in the way user configured + ;; things: directly through display buffer (via pop-t or otherwise through + ;; mu4e-window. + (if (eq mu4e-compose-switch 'display-buffer) + (pop-to-buffer draft-buffer) + (mu4e-display-buffer draft-buffer 'do-select)) + ;; prepare possible message actions (such as cleaning-up) + (mu4e--prepare-post oldframe oldwinconf) + draft-buffer)) + +(defun mu4e--draft-with-parent (compose-type parent compose-func) + "Draft a message based on some parent message. +COMPOSE-TYPE, COMPOSE-FUNC and PARENT are as in `mu4e--draft', +but note the different order." + (mu4e--draft + compose-type + (lambda () + (let ( ;; only needed for Fwd. Gnus has a bad default. + (message-make-forward-subject-function + (list #'message-forward-subject-fwd))) + (insert (mu4e--decoded-message parent)) + ;; let's make sure we don't use message-reply-headers from + ;; some unrelated message. + (setq message-reply-headers nil) + (funcall compose-func))) + parent)) + +(provide 'mu4e-draft) diff --git a/mu4e/mu4e-folders.el b/mu4e/mu4e-folders.el new file mode 100644 index 0000000..330264a --- /dev/null +++ b/mu4e/mu4e-folders.el @@ -0,0 +1,302 @@ +;;; mu4e-folders.el --- Dealing with maildirs & folders -*- lexical-binding: t -*- + +;; Copyright (C) 2021-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Dealing with maildirs & folders + +;;; Code: +(require 'mu4e-helpers) +(require 'mu4e-context) +(require 'mu4e-server) + +;;; Customization +(defgroup mu4e-folders nil + "Special folders." + :group 'mu4e) + +(defcustom mu4e-drafts-folder "/drafts" + "Folder for draft messages, relative to the root maildir. +For instance, \"/drafts\". Instead of a string, may also be a +function that takes a message (a msg plist, see +`mu4e-message-field'), and returns a folder. Note, the message +parameter refers to the original message being replied to / being +forwarded / re-edited and is nil otherwise. `mu4e-drafts-folder' +is only evaluated once." + :type '(choice + (string :tag "Folder name") + (function :tag "Function return folder name")) + :group 'mu4e-folders) + +(defcustom mu4e-refile-folder "/archive" + "Folder for refiling messages, relative to the root maildir. +For instance \"/Archive\". Instead of a string, may also be a +function that takes a message (a msg plist, see +`mu4e-message-field'), and returns a folder. Note that the +message parameter refers to the message-at-point." + :type '(choice + (string :tag "Folder name") + (function :tag "Function return folder name")) + :group 'mu4e-folders) + +(defcustom mu4e-sent-folder "/sent" + "Folder for sent messages, relative to the root maildir. +For instance, \"/Sent Items\". Instead of a string, may also be a +function that takes a message (a msg plist, see +`mu4e-message-field'), and returns a folder. Note that the +message parameter refers to the original message being replied to +/ being forwarded / re-edited, and is nil otherwise." + :type '(choice + (string :tag "Folder name") + (function :tag "Function return folder name")) + :group 'mu4e-folders) + +(defcustom mu4e-trash-folder "/trash" + "Folder for trashed messages, relative to the root maildir. +For instance, \"/trash\". Instead of a string, may also be a +function that takes a message (a msg plist, see +`mu4e-message-field'), and returns a folder. When using +`mu4e-trash-folder' in the headers view (when marking messages +for trash). Note that the message parameter refers to the +message-at-point. When using it when composing a message (see +`mu4e-sent-messages-behavior'), this refers to the original +message being replied to / being forwarded / re-edited, and is +nil otherwise." + :type '(choice + (string :tag "Folder name") + (function :tag "Function return folder name")) + :group 'mu4e-folders) + +(defcustom mu4e-maildir-shortcuts nil + "A list of maildir shortcuts. +This makes it possible to quickly go to a particular +maildir (folder), or quickly moving messages to them (e.g., for +archiving or refiling). + +Each of the list elements is a plist with at least: +`:maildir' - the maildir for the shortcut (e.g. \"/archive\") +`:key' - the shortcut key. + +Optionally, you can add the following: +`:name' - name of the maildir to be displayed in main-view. +`:hide' - if t, the shortcut is hidden from the main-view and +speedbar. +`:hide-unread' - do not show the counts of unread/total number + of matches for the maildir in the main-view, and is implied +from `:hide'. + +For backward compatibility, an older form is recognized as well: + + (maildir . key), where MAILDIR is a maildir (such as +\"/archive/\"), and key is a single character. + +You can use these shortcuts in the headers and view buffers, for +example with `mu4e-mark-for-move-quick' (or \"m\", by default) or +`mu4e-jump-to-maildir' (or \"j\", by default), followed by the +designated shortcut character for the maildir. + +Unlike in search queries, folder names with spaces in them must +NOT be quoted, since mu4e does this for you." + :type '(choice + (alist :key-type (string :tag "Maildir") + :value-type character + :tag "Alist (old format)") + (repeat (plist + :key-type (choice (const :tag "Maildir" :maildir) + (const :tag "Shortcut" :key) + (const :tag "Name of maildir" :name) + (const :tag "Hide from main view" :hide) + (const :tag "Do not count" :hide-unread)) + :tag "Plist (new format)"))) + :version "1.3.9" + :group 'mu4e-folders) + +(defcustom mu4e-maildir-initial-input "/" + "Initial input for `mu4e-completing-completing-read' function." + :type 'string + :group 'mu4e-folders) + +(defcustom mu4e-maildir-info-delimiter + (if (member system-type '(ms-dos windows-nt cygwin)) + ";" ":") + "Separator character between message identifier and flags. +It defaults to ':' on most platforms, except on Windows, where it +is not allowed and we use ';' for compatibility with mbsync, +offlineimap and other programs." + :type 'string + :group 'mu4e-folders) + +(defcustom mu4e-attachment-dir (expand-file-name "~/") + "Default directory for attaching and saving attachments. + +This can be either a string (a file system path), or a function +that takes a filename and the mime-type as arguments, and returns +the attachment dir. See Info node `(mu4e) Attachments' for +details. + +When this called for composing a message, both filename and +mime-type are nil." + :type 'directory + :group 'mu4e-folders + :safe 'stringp) + +(defvar mu4e-maildir-list nil + "Cached list of maildirs.") + + +(defun mu4e-maildir-shortcuts () + "Get `mu4e-maildir-shortcuts' in the (new) format. +Converts from the old format if needed." + (seq-map (lambda (item) ;; convert from old format? + (if (and (consp item) (not (consp (cdr item)))) + `(:maildir ,(car item) :key ,(cdr item)) + item)) + mu4e-maildir-shortcuts)) + +;; the standard folders can be functions too +(defun mu4e--get-folder (foldervar msg) + "Within the mu-context of MSG, get message folder FOLDERVAR. +If FOLDER is a string, return it, if it is a function, evaluate +this function with MSG as parameter which may be nil, and return +the result." + (unless (member foldervar + '(mu4e-sent-folder mu4e-drafts-folder + mu4e-trash-folder mu4e-refile-folder)) + (mu4e-error "Folder must be one of mu4e-(sent|drafts|trash|refile)-folder")) + ;; get the value with the vars for the relevants context let-bound + (with-mu4e-context-vars (mu4e-context-determine msg nil) + (let* ((folder (symbol-value foldervar)) + (val + (cond + ((stringp folder) folder) + ((functionp folder) (funcall folder msg)) + (t (mu4e-error "Unsupported type for %S" folder))))) + (or val (mu4e-error "%S evaluates to nil" foldervar))))) + +(defun mu4e-get-drafts-folder (&optional msg) + "Get the drafts folder, optionally based on MSG. +See `mu4e-drafts-folder'." (mu4e--get-folder 'mu4e-drafts-folder msg)) + +(defun mu4e-get-refile-folder (&optional msg) + "Get the folder for refiling, optionally based on MSG. +See `mu4e-refile-folder'." (mu4e--get-folder 'mu4e-refile-folder msg)) + +(defun mu4e-get-sent-folder (&optional msg) + "Get the sent folder, optionally based on MSG. +See `mu4e-sent-folder'." (mu4e--get-folder 'mu4e-sent-folder msg)) + +(defun mu4e-get-trash-folder (&optional msg) + "Get the trash folder, optionally based on MSG. +See `mu4e-trash-folder'." (mu4e--get-folder 'mu4e-trash-folder msg)) + +;;; Maildirs +(defun mu4e--guess-maildir (path) + "Guess the maildir for PATH, or nil if cannot find it." + (let ((idx (string-match (mu4e-root-maildir) path))) + (when (and idx (zerop idx)) + (replace-regexp-in-string + (mu4e-root-maildir) + "" + (expand-file-name + (mu4e-join-paths path ".." "..")))))) + +(defun mu4e-create-maildir-maybe (dir) + "Offer to create maildir DIR if it does not exist yet. +Return t if it already exists or (after asking) an attempt has been +to create it; otherwise return nil." + (let ((seems-to-exist (file-directory-p dir))) + (when (or seems-to-exist + (yes-or-no-p (mu4e-format "%s does not exist yet. Create now?" dir))) + ;; even when the maildir already seems to exist, call mkdir for a deeper + ;; check. However only get an update when the maildir is totally new. + (mu4e--server-mkdir dir (not seems-to-exist)) + t))) + +(defun mu4e-get-maildirs () + "Get maildirs under `mu4e-maildir'." + mu4e-maildir-list) + +(defun mu4e-ask-maildir (prompt) + "Ask the user for a maildir (using PROMPT). + +If the special shortcut \"o\" (for _o_ther) is used, or +if (mu4e-maildir-shortcuts) evaluates to nil, let user choose +from all maildirs under `mu4e-maildir'." + (let* ((options + (seq-map (lambda (md) + (cons + (format "%c%s" (plist-get md :key) + (or (plist-get md :name) + (plist-get md :maildir))) + (plist-get md :maildir))) + (mu4e-filter-single-key (mu4e-maildir-shortcuts)))) + (response + (if (not options) + 'other + (mu4e-read-option prompt + (append options + '(("oOther..." . other))))))) + (substring-no-properties + (if (eq response 'other) + (progn + (funcall mu4e-completing-read-function prompt + (mu4e-get-maildirs) nil nil + mu4e-maildir-initial-input)) + response)))) + +(defun mu4e-ask-maildir-check-exists (prompt) + "Like `mu4e-ask-maildir', PROMPT for existence of the maildir. +Offer to create it if it does not exist yet." + (let* ((mdir (mu4e-ask-maildir prompt)) + (fullpath (mu4e-join-paths (mu4e-root-maildir) mdir))) + (unless (file-directory-p fullpath) + (and (yes-or-no-p + (mu4e-format "%s does not exist. Create now?" fullpath)) + (mu4e--server-mkdir fullpath))) + mdir)) + +;; mu4e-attachment-dir is either a string or a function that takes a +;; filename and the mime-type as argument, either (or both) which can +;; be nil + +(defun mu4e-determine-attachment-dir (&optional fname mimetype) + "Get the target-directory for attachments. + +This is based on the variable `mu4e-attachment-dir', which is either: +- if is a string, used it as-is +- a function taking two string parameters, both of which can be nil: + (1) a filename or a URL + (2) a mime-type (such as \"text/plain\"." + (let ((dir + (cond + ((stringp mu4e-attachment-dir) + mu4e-attachment-dir) + ((functionp mu4e-attachment-dir) + (funcall mu4e-attachment-dir fname mimetype)) + (t + (mu4e-error "Unsupported type for mu4e-attachment-dir" ))))) + (if dir + (expand-file-name dir) + (mu4e-error "Mu4e-attachment-dir evaluates to nil")))) + +(provide 'mu4e-folders) +;;; mu4e-folders.el ends here diff --git a/mu4e/mu4e-headers.el b/mu4e/mu4e-headers.el new file mode 100644 index 0000000..11d1dea --- /dev/null +++ b/mu4e/mu4e-headers.el @@ -0,0 +1,1648 @@ +;;; mu4e-headers.el --- Message headers -*- lexical-binding: t; coding:utf-8 -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; In this file are function related mu4e-headers-mode, to creating the list of +;; one-line descriptions of emails, aka 'headers' (not to be confused with +;; headers like 'To:' or 'Subject:') + +;;; Code: + +(require 'cl-lib) +(require 'fringe) +(require 'hl-line) +(require 'mailcap) +(require 'mule-util) ;; seems _some_ people need this for + ;; truncate-string-ellipsis + +(require 'mu4e-update) + + ;; utility functions +(require 'mu4e-server) +(require 'mu4e-vars) +(require 'mu4e-mark) +(require 'mu4e-context) +(require 'mu4e-contacts) +(require 'mu4e-search) +(require 'mu4e-compose) +(require 'mu4e-actions) +(require 'mu4e-message) +(require 'mu4e-lists) +(require 'mu4e-update) +(require 'mu4e-folders) +(require 'mu4e-thread) + +(declare-function mu4e-view "mu4e-view") +(declare-function mu4e--main-view "mu4e-main") + + +;;; Configuration + +(defgroup mu4e-headers nil + "Settings for the headers view." + :group 'mu4e) + +(defcustom mu4e-headers-fields + '( (:human-date . 12) + (:flags . 6) + (:mailing-list . 10) + (:from . 22) + (:subject . nil)) + "A list of header fields to show in the headers buffer. +Each element has the form (HEADER . WIDTH), where HEADER is one of +the available headers (see `mu4e-header-info') and WIDTH is the +respective width in characters. + +A width of nil means \"unrestricted\", and this is best reserved +for the rightmost (last) field. Note that emacs may become very +slow with excessively long lines (1000s of characters), so if you +regularly get such messages, you want to avoid fields with nil +altogether." + :type `(repeat (cons (choice + ,@(mapcar (lambda (h) + (list 'const :tag + (plist-get (cdr h) :help) + (car h))) + mu4e-header-info) + (restricted-sexp + :tag "User-specified header" + :match-alternatives (mu4e--headers-header-p))) + (choice (integer :tag "width") + (const :tag "unrestricted width" nil)))) + :group 'mu4e-headers) + +(defun mu4e--headers-header-p (symbol) + "Is symbol a valid mu4e header? +This means its either one of the build-in or user-specified headers." + (assoc symbol (append mu4e-header-info mu4e-header-info-custom))) + +(defcustom mu4e-headers-date-format "%x" + "Date format to use in the headers view. +In the format of `format-time-string'." + :type 'string + :group 'mu4e-headers) + +(defcustom mu4e-headers-time-format "%X" + "Time format to use in the headers view. +In the format of `format-time-string'." + :type 'string + :group 'mu4e-headers) + +(defcustom mu4e-headers-long-date-format "%c" + "Date format to use in the headers view tooltip. +In the format of `format-time-string'." + :type 'string + :group 'mu4e-headers) + +(defcustom mu4e-headers-precise-alignment nil + "When set, use precise (but relatively slow) alignment for columns. +By default, do it in a slightly inaccurate but faster way. To get +an idea about the difference, In some tests, the rendering time +was around 5.8 ms per messages for precise alignment, versus 3.3 +for non-precise aligment (for 445 messages)." + :type 'boolean + :group 'mu4e-headers) + +(defcustom mu4e-headers-auto-update t + "Whether to automatically update the current headers buffer if an +indexing operation showed changes." + :type 'boolean + :group 'mu4e-headers) + +(defcustom mu4e-headers-advance-after-mark t + "With this option set to non-nil, automatically advance to the +next mail after marking a message in header view." + :type 'boolean + :group 'mu4e-headers) + + +(defcustom mu4e-headers-visible-flags + '(draft flagged new passed replied trashed attach encrypted signed + list personal) + "An ordered list of flags to show in the headers buffer. +Each element is a symbol in the list. + +By default, we leave out `unread' and `seen', since those are +mostly covered by `new', and the display gets cluttered otherwise." + :type '(set + (const :tag "Draft" draft) + (const :tag "Flagged" flagged) + (const :tag "New" new) + (const :tag "Passed" passed) + (const :tag "Replied" replied) + (const :tag "Seen" seen) + (const :tag "Trashed" trashed) + (const :tag "Attach" attach) + (const :tag "Encrypted" encrypted) + (const :tag "Signed" signed) + (const :tag "List" list) + (const :tag "Personal" personal) + (const :tag "Calendar" calendar)) + :group 'mu4e-headers) + +(defcustom mu4e-headers-found-hook nil + "Hook run just *after* all of the headers for the last search +query have been received and are displayed." + :type 'hook + :group 'mu4e-headers) + +;;; Public variables +(defcustom mu4e-headers-from-or-to-prefix '("" . "To ") + "Prefix for the :from-or-to field when it is showing, + respectively, From: or To:. It is a cons cell with the car + element being the From: prefix, the cdr element the To: prefix." + :type '(cons string string) + :group 'mu4e-headers) + +;;;; Fancy marks + +;; marks for headers of the form; each is a cons-cell (basic . fancy) +;; each of which is basic ascii char and something fancy, respectively +;; by default, we some conservative marks, even when 'fancy' +;; so they're less likely to break if people don't have certain fonts. +;; However, if you want to be really 'fancy', you could use something like +;; the following; esp. with a newer Emacs with color-icon support. +;; (setq +;; mu4e-headers-draft-mark '("D" . "💈") +;; mu4e-headers-flagged-mark '("F" . "📍") +;; mu4e-headers-new-mark '("N" . "🔥") +;; mu4e-headers-passed-mark '("P" . "❯") +;; mu4e-headers-replied-mark '("R" . "❮") +;; mu4e-headers-seen-mark '("S" . "☑") +;; mu4e-headers-trashed-mark '("T" . "💀") +;; mu4e-headers-attach-mark '("a" . "📎") +;; mu4e-headers-encrypted-mark '("x" . "🔒") +;; mu4e-headers-signed-mark '("s" . "🔑") +;; mu4e-headers-unread-mark '("u" . "⎕") +;; mu4e-headers-list-mark '("l" . "🔈") +;; mu4e-headers-personal-mark '("p" . "👨") +;; mu4e-headers-calendar-mark '("c" . "📅")) + + +(defvar mu4e-headers-draft-mark '("D" . "⚒") "Draft.") +(defvar mu4e-headers-flagged-mark '("F" . "✚") "Flagged.") +(defvar mu4e-headers-new-mark '("N" . "✱") "New.") +(defvar mu4e-headers-passed-mark '("P" . "❯") "Passed (fwd).") +(defvar mu4e-headers-replied-mark '("R" . "❮") "Replied.") +(defvar mu4e-headers-seen-mark '("S" . "✔") "Seen.") +(defvar mu4e-headers-trashed-mark '("T" . "⏚") "Trashed.") +(defvar mu4e-headers-attach-mark '("a" . "⚓") "W/ attachments.") +(defvar mu4e-headers-encrypted-mark '("x" . "⚴") "Encrypted.") +(defvar mu4e-headers-signed-mark '("s" . "☡") "Signed.") +(defvar mu4e-headers-unread-mark '("u" . "⎕") "Unread.") +(defvar mu4e-headers-list-mark '("l" . "Ⓛ") "Mailing list.") +(defvar mu4e-headers-personal-mark '("p" . "Ⓟ") "Personal.") +(defvar mu4e-headers-calendar-mark '("c" . "Ⓒ") "Calendar invitation.") + + +;;;; Graph drawing + +(defvar mu4e-headers-thread-mark-as-orphan 'first + "Define which messages should be prefixed with the orphan mark. +`all' marks all the messages without a parent as orphan, `first' +only marks the first message in the thread.") + +(defvar mu4e-headers-thread-root-prefix '("* " . "□ ") + "Prefix for root messages.") +(defvar mu4e-headers-thread-child-prefix '("|>" . "│ ") + "Prefix for messages in sub threads that do have a following sibling.") +(defvar mu4e-headers-thread-first-child-prefix '("o " . "⚬ ") + "Prefix for the first child messages in a sub thread.") +(defvar mu4e-headers-thread-last-child-prefix '("L" . "└ ") + "Prefix for messages in sub threads that do not have a following sibling.") +(defvar mu4e-headers-thread-connection-prefix '("|" . "│ ") + "Prefix to connect sibling messages that do not follow each other. +Must have the same length as `mu4e-headers-thread-blank-prefix'.") +(defvar mu4e-headers-thread-blank-prefix '(" " . " ") + "Prefix to separate non connected messages. +Must have the same length as `mu4e-headers-thread-connection-prefix'.") +(defvar mu4e-headers-thread-orphan-prefix '("<>" . "♢ ") + "Prefix for orphan messages with siblings.") +(defvar mu4e-headers-thread-single-orphan-prefix '("<>" . "♢ ") + "Prefix for orphan messages with no siblings.") +(defvar mu4e-headers-thread-duplicate-prefix '("=" . "≡ ") + "Prefix for duplicate messages.") + +;;;; Various + +(defcustom mu4e-headers-actions + '( ("capture message" . mu4e-action-capture-message) + ("browse online archive" . mu4e-action-browse-list-archive) + ("show this thread" . mu4e-action-show-thread)) + "List of actions to perform on messages in the headers list. +The actions are cons-cells of the form (NAME . FUNC) where: +* NAME is the name of the action (e.g. \"Count lines\") +* FUNC is a function which receives a message plist as an argument. + +The first character of NAME is used as the shortcut." + :group 'mu4e-headers + :type '(alist :key-type string :value-type function)) + +(defvar mu4e-headers-custom-markers + '(("Older than" + (lambda (msg date) (time-less-p (mu4e-msg-field msg :date) date)) + (lambda () (mu4e-get-time-date "Match messages before: "))) + ("Newer than" + (lambda (msg date) (time-less-p date (mu4e-msg-field msg :date))) + (lambda () (mu4e-get-time-date "Match messages after: "))) + ("Bigger than" + (lambda (msg bytes) (> (mu4e-msg-field msg :size) (* 1024 bytes))) + (lambda () (read-number "Match messages bigger than (Kbytes): ")))) + "List of custom markers -- functions to mark message that match +some custom function. Each of the list members has the following +format: + (NAME PREDICATE-FUNC PARAM-FUNC) +* NAME is the name of the predicate function, and the first +character is the shortcut (so keep those unique). +* PREDICATE-FUNC is a function that takes two parameters, MSG +and (optionally) PARAM, and should return non-nil when there's a +match. +* PARAM-FUNC is function that is evaluated once, and its value is +then passed to PREDICATE-FUNC as PARAM. This is useful for +getting user-input.") +;;; Internal variables/constants + +;; docid cookies +(defconst mu4e~headers-docid-pre "\376" + "Each header starts (invisibly) with the `mu4e~headers-docid-pre', +followed by the docid, followed by `mu4e~headers-docid-post'.") +(defconst mu4e~headers-docid-post "\377" + "Each header starts (invisibly) with the `mu4e~headers-docid-pre', +followed by the docid, followed by `mu4e~headers-docid-post'.") + + +(defvar mu4e~headers-search-start nil) +(defvar mu4e~headers-render-start nil) +(defvar mu4e~headers-render-time nil) + +(defvar mu4e-headers-report-render-time nil + "If non-nil, report on the time it took to render the messages. +This is mostly useful for profiling.") + +(defvar mu4e~headers-hidden 0 + "Number of headers hidden due to `mu4e-headers-hide-predicate'.") + + +;;; Clear + +(defun mu4e~headers-clear (&optional text) + "Clear the headers buffer and related data structures. +Optionally, show TEXT." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (setq mu4e~headers-render-start (float-time) + mu4e~headers-hidden 0) + (let ((inhibit-read-only t)) + (with-current-buffer (mu4e-get-headers-buffer) + (mu4e--mark-clear) + (remove-overlays) + (erase-buffer) + (when text + (goto-char (point-min)) + (insert (propertize text 'face 'mu4e-system-face 'intangible t))))))) + +;;; Misc + +(defun mu4e~headers-contact-str (contacts) + "Turn the list of contacts CONTACTS (with elements (NAME . EMAIL) +into a string." + (mapconcat + (lambda (contact) + (let ((name (mu4e-contact-name contact)) + (email (mu4e-contact-email contact))) + (or name email "?"))) contacts ", ")) + +(defun mu4e~headers-thread-prefix-map (type) + "Return the thread prefix based on the symbol TYPE." + (let ((get-prefix + (lambda (cell) + (if mu4e-use-fancy-chars (cdr cell) (car cell))))) + (propertize + (cl-case type + (child (funcall get-prefix mu4e-headers-thread-child-prefix)) + (first-child (funcall get-prefix mu4e-headers-thread-first-child-prefix)) + (last-child (funcall get-prefix mu4e-headers-thread-last-child-prefix)) + (connection (funcall get-prefix mu4e-headers-thread-connection-prefix)) + (blank (funcall get-prefix mu4e-headers-thread-blank-prefix)) + (orphan (funcall get-prefix mu4e-headers-thread-orphan-prefix)) + (single-orphan (funcall get-prefix + mu4e-headers-thread-single-orphan-prefix)) + (duplicate (funcall get-prefix mu4e-headers-thread-duplicate-prefix)) + (t "?")) + 'face 'mu4e-thread-fold-face))) + + +;; headers in the buffer are prefixed by an invisible string with the docid +;; followed by an EOT ('end-of-transmission', \004, ^D) non-printable ascii +;; character. this string also has a text-property with the docid. the former +;; is used for quickly finding a certain header, the latter for retrieving the +;; docid at point without string matching etc. + +(defun mu4e~headers-docid-pos (docid) + "Return the pos of the beginning of the line with the header with +docid DOCID, or nil if it cannot be found." + (let ((pos)) + (save-excursion + (setq pos (mu4e~headers-goto-docid docid))) + pos)) + +(defun mu4e~headers-docid-cookie (docid) + "Create an invisible string containing DOCID; this is to be used +at the beginning of lines to identify headers." + (propertize (format "%s%d%s" + mu4e~headers-docid-pre docid mu4e~headers-docid-post) + 'docid docid 'invisible t));; + +(defun mu4e~headers-docid-at-point (&optional point) + "Get the docid for the header at POINT, or at current (point) if +nil. Returns the docid, or nil if there is none." + (save-excursion + (when point + (goto-char point)) + (get-text-property (line-beginning-position) 'docid))) + + + +(defun mu4e~headers-goto-docid (docid &optional to-mark) + "Go to the beginning of the line with the header with docid +DOCID, or nil if it cannot be found. If the optional TO-MARK is +non-nil, go to the point directly *after* the docid-cookie instead +of the beginning of the line." + (let ((oldpoint (point)) (newpoint)) + (goto-char (point-min)) + (setq newpoint + (search-forward (mu4e~headers-docid-cookie docid) nil t)) + (unless to-mark + (if (null newpoint) + (goto-char oldpoint) ;; not found; restore old pos + (progn + (beginning-of-line) ;; found, move to beginning of line + (setq newpoint (point))))) + newpoint)) ;; return the point, or nil if not found + +(defun mu4e~headers-field-for-docid (docid field) + "Get FIELD (a symbol, see `mu4e-headers-names') for the message +with DOCID which must be present in the headers buffer." + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-message-field (mu4e-message-at-point) field)))) + + +;; In order to print a thread tree with all the message connections, +;; it's necessary to keep track of all sub levels that still have +;; following messages. For each level, mu4e~headers-thread-state keeps +;; the value t for a connection or nil otherwise. +(defvar-local mu4e~headers-thread-state '()) + +(defun mu4e~headers-thread-prefix (thread) + "Calculate the thread prefix based on thread info THREAD." + (when thread + (let* ((prefix "") + (level (plist-get thread :level)) + (has-child (plist-get thread :has-child)) + (first-child (plist-get thread :first-child)) + (last-child (plist-get thread :last-child)) + (orphan (plist-get thread :orphan)) + (single-orphan(and orphan first-child last-child)) + (duplicate (plist-get thread :duplicate))) + ;; Do not prefix root messages. + (if (= level 0) + (setq mu4e~headers-thread-state '())) + (if (> level 0) + (let* ((length (length mu4e~headers-thread-state)) + (padding (make-list (max 0 (- level length)) nil))) + ;; Trim and pad the state to ensure a message will + ;; always be shown with the correct indentation, even if + ;; a broken thread is returned. It's trimmed to level-1 + ;; because the current level has always an connection + ;; and it used a special formatting. + (setq mu4e~headers-thread-state + (cl-subseq (append mu4e~headers-thread-state padding) + 0 (- level 1))) + ;; Prepare the thread prefix. + (setq prefix + (concat + ;; Current mu4e~headers-thread-state, composed by + ;; connections or blanks. + (mapconcat + (lambda (s) + (mu4e~headers-thread-prefix-map + (if s 'connection 'blank))) + mu4e~headers-thread-state "") + ;; Current entry. + (mu4e~headers-thread-prefix-map + (if single-orphan 'single-orphan + (if (and orphan + (or first-child + (not (eq mu4e-headers-thread-mark-as-orphan + 'first)))) + 'orphan + (if last-child 'last-child + (if first-child 'first-child + 'child))))))))) + ;; If a new sub-thread will follow (has-child) and the current + ;; one is still not done (not last-child), then a new + ;; connection needs to be added to the tree-state. It's not + ;; necessary to a blank (nil), because padding will handle + ;; that. + (if (and has-child (not last-child)) + (setq mu4e~headers-thread-state + (append mu4e~headers-thread-state '(t)))) + ;; Return the thread prefix. + (format "%s%s" + prefix + (if duplicate + (mu4e~headers-thread-prefix-map 'duplicate) ""))))) + +(defun mu4e~headers-flags-str (flags) + "Get a display string for FLAGS. +Note that `mu4e-flags-to-string' is for internal use only; this +function is for display. (This difference is significant, since +internally, the Maildir spec determines what the flags look like, +while our display may be different)." + (or (mapconcat + (lambda (flag) + (when (member flag mu4e-headers-visible-flags) + (if-let* ((mark (intern-soft + (format "mu4e-headers-%s-mark" (symbol-name flag)))) + (cell (symbol-value mark))) + (if mu4e-use-fancy-chars (cdr cell) (car cell)) + ""))) + flags "") + "")) + +;;; Special headers + +(defun mu4e~headers-from-or-to (msg) + "Get the From: address from MSG if not one of user's; otherwise get To:. +When the from address for message MSG is one of the the user's +addresses, (as per `mu4e-personal-address-p'), show the To +address. Otherwise, show the From address, prefixed with the +appropriate `mu4e-headers-from-or-to-prefix'." + (let* ((from1 (car-safe (mu4e-message-field msg :from))) + (from1-addr (and from1 (mu4e-contact-email from1))) + (is-user (and from1-addr (mu4e-personal-address-p from1-addr)))) + (if is-user + (concat (cdr mu4e-headers-from-or-to-prefix) + (mu4e~headers-contact-str (mu4e-message-field msg :to))) + (concat (car mu4e-headers-from-or-to-prefix) + (mu4e~headers-contact-str (mu4e-message-field msg :from)))))) + +(defun mu4e~headers-human-date (msg) + "Show a \"human\" date for MSG. +If the date is today, show the time, otherwise, show the date. +The formats used for date and time are `mu4e-headers-date-format' +and `mu4e-headers-time-format'." + (let ((date (mu4e-msg-field msg :date))) + (if (equal date '(0 0 0)) + "None" + (let ((day1 (decode-time date)) + (day2 (decode-time (current-time)))) + (if (and + (eq (nth 3 day1) (nth 3 day2)) ;; day + (eq (nth 4 day1) (nth 4 day2)) ;; month + (eq (nth 5 day1) (nth 5 day2))) ;; year + (format-time-string mu4e-headers-time-format date) + (format-time-string mu4e-headers-date-format date)))))) + +(defun mu4e~headers-thread-subject (msg) + "Get the subject for MSG if it is the first one in a thread. +Otherwise, return the thread-prefix without the subject-text. In +other words, show the subject of a thread only once, similar to +e.g. \"mutt\"." + (let* ((tinfo (mu4e-message-field msg :meta)) + (subj (mu4e-msg-field msg :subject))) + (concat ;; prefix subject with a thread indicator + (mu4e~headers-thread-prefix tinfo) + (if (plist-get tinfo :thread-subject) + (truncate-string-to-width subj 600) "")))) + +(defun mu4e~headers-mailing-list (list) + "Get some identifier for the mailing list." + (if list + (propertize (mu4e-get-mailing-list-shortname list) 'help-echo list) + "")) + +(defsubst mu4e~headers-custom-field-value (msg field) + "Show some custom header field, or raise an error if it is not +found." + (let* ((item (or (assoc field mu4e-header-info-custom) + (mu4e-error "field %S not found" field))) + (func (or (plist-get (cdr-safe item) :function) + (mu4e-error "no :function defined for field %S %S" + field (cdr item))))) + (funcall func msg))) + +(defun mu4e~headers-field-value (msg field) + (let ((val (mu4e-message-field msg field))) + (cl-case field + (:subject + (concat ;; prefix subject with a thread indicator + (mu4e~headers-thread-prefix (mu4e-message-field msg :meta)) + ;; "["(plist-get (mu4e-message-field msg :meta) :path) "] " + ;; work-around: emacs' display gets really slow when lines are too long; + ;; so limit subject length to 600 + (truncate-string-to-width val 600))) + (:thread-subject ;; if not searching threads, fall back to :subject + (if mu4e-search-threads + (mu4e~headers-thread-subject msg) + (mu4e~headers-field-value msg :subject))) + ((:maildir :path :message-id) val) + ((:to :from :cc :bcc) (mu4e~headers-contact-str val)) + ;; if we (ie. `user-mail-address' is the 'From', show + ;; 'To', otherwise show From + (:from-or-to (mu4e~headers-from-or-to msg)) + (:date (format-time-string mu4e-headers-date-format val)) + (:list (or val "")) + (:mailing-list (mu4e~headers-mailing-list (mu4e-msg-field msg :list))) + (:human-date (propertize (mu4e~headers-human-date msg) + 'help-echo (format-time-string + mu4e-headers-long-date-format + (mu4e-msg-field msg :date)))) + (:flags (propertize (mu4e~headers-flags-str val) + 'help-echo (format "%S" val))) + (:tags (propertize (mapconcat 'identity val ", "))) + (:size (mu4e-display-size val)) + (t (mu4e~headers-custom-field-value msg field))))) + +(defsubst mu4e~headers-truncate-field-fast (val width) + "Truncate VAL to WIDTH. Fast and somewhat inaccurate." + (if width + (truncate-string-to-width val width 0 ?\s truncate-string-ellipsis) + val)) + +(defun mu4e~headers-truncate-field-precise (field val width) + "Return VAL truncated to one less than WIDTH, with a trailing +space propertized with a `display' text property which expands to + the correct column for display." + (when width + (let ((end-col (cl-loop for (f . w) in mu4e-headers-fields + sum w + until (equal f field)))) + (setq val (string-trim-right val)) + (if (> width (length val)) + (setq val (concat val " ")) + (setq val + (concat + (truncate-string-to-width val (1- width) 0 ?\s t) + " "))) + (put-text-property (1- (length val)) + (length val) + 'display + `(space . (:align-to ,end-col)) + val))) + val) + +(defsubst mu4e~headers-truncate-field (field val width) + "Truncate VAL to WIDTH." + (if mu4e-headers-precise-alignment + (mu4e~headers-truncate-field-precise field val width) + (mu4e~headers-truncate-field-fast val width))) + +(defsubst mu4e~headers-field-handler (f-w msg) + "Create a description of the field of MSG described by F-W." + (let* ((field (car f-w)) + (width (cdr f-w)) + (val (mu4e~headers-field-value msg field)) + (val (and val + (if width + (mu4e~headers-truncate-field field val width) + val)))) + val)) + +(defsubst mu4e~headers-apply-flags (msg fieldval) + "Adjust FIELDVAL's face property based on flags in MSG." + (let* ((flags (plist-get msg :flags)) + (meta (plist-get msg :meta)) + (face (cond + ((memq 'trashed flags) 'mu4e-trashed-face) + ((memq 'draft flags) 'mu4e-draft-face) + ((or (memq 'unread flags) (memq 'new flags)) + 'mu4e-unread-face) + ((memq 'flagged flags) 'mu4e-flagged-face) + ((plist-get meta :related) 'mu4e-related-face) + ((memq 'replied flags) 'mu4e-replied-face) + ((memq 'passed flags) 'mu4e-forwarded-face) + (t 'mu4e-header-face)))) + (add-face-text-property 0 (length fieldval) face t fieldval) + fieldval)) + +(defsubst mu4e~message-header-line (msg) + "Return a propertized description of message MSG suitable for +displaying in the header view." + (if (and mu4e-search-hide-enabled mu4e-search-hide-predicate + (funcall mu4e-search-hide-predicate msg)) + (progn + (cl-incf mu4e~headers-hidden) + nil) + (progn + (mu4e~headers-apply-flags + msg + (mapconcat (lambda (f-w) (mu4e~headers-field-handler f-w msg)) + mu4e-headers-fields " "))))) + +(defsubst mu4e~headers-insert-header (msg pos) + "Insert a header for MSG at point POS." + (when-let ((line (mu4e~message-header-line msg)) + (docid (plist-get msg :docid))) + (goto-char pos) + (insert + (propertize + (concat + (mu4e~headers-docid-cookie docid) + mu4e--mark-fringe line "\n") + 'docid docid 'msg msg)))) + +(defun mu4e~headers-remove-header (docid &optional ignore-missing) + "Remove header with DOCID at point. +When IGNORE-MISSING is non-nill, don't raise an error when the +docid is not found." + (with-current-buffer (mu4e-get-headers-buffer) + (if (mu4e~headers-goto-docid docid) + (let ((inhibit-read-only t)) + (delete-region (line-beginning-position) (line-beginning-position 2))) + (unless ignore-missing + (mu4e-error "Cannot find message with docid %S" docid))))) + + +;;; Handler functions + +;; next are a bunch of handler functions; those will be called from mu4e~proc in +;; response to output from the server process + +(defun mu4e~headers-view-handler (msg) + "Handler function for displaying a message." + (mu4e-view msg)) + +(defun mu4e~headers-view-this-message-p (docid) + "Is DOCID currently being viewed?" + (mu4e-get-view-buffers + (lambda (_) (eq docid (plist-get mu4e--view-message :docid))))) + +;; note: this function is very performance-sensitive +(defun mu4e~headers-append-handler (msglst) + "Append one-line descriptions of messages in MSGLIST. +Do this at the end of the headers-buffer." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (with-current-buffer (mu4e-get-headers-buffer) + (save-excursion + (let ((inhibit-read-only t)) + (seq-do + (lambda (msg) + (mu4e~headers-insert-header msg (point-max))) + msglst)))))) + + +(defun mu4e~headers-update-handler (msg is-move maybe-view) + "Update handler, will be called when a message has been updated +in the database. This function will update the current list of +headers." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (with-current-buffer (mu4e-get-headers-buffer) + (let* ((docid (mu4e-message-field msg :docid)) + (initial-message-at-point (mu4e~headers-docid-at-point)) + (initial-column (current-column)) + (inhibit-read-only t) + (point (mu4e~headers-docid-pos docid)) + (markinfo (gethash docid mu4e--mark-map))) + (when point ;; is the message present in this list? + + ;; if it's marked, unmark it now + (when (mu4e-mark-docid-marked-p docid) + (mu4e-mark-set 'unmark)) + + ;; re-use the thread info from the old one; this is needed because + ;; *update* messages don't have thread info by themselves (unlike + ;; search results) + ;; since we still have the search results, re-use + ;; those + (plist-put msg :meta + (mu4e~headers-field-for-docid docid :meta)) + + ;; first, remove the old one (otherwise, we'd have two headers with + ;; the same docid... + (mu4e~headers-remove-header docid t) + + ;; if we're actually viewing this message (in mu4e-view mode), we + ;; update it; that way, the flags can be updated, as well as the path + ;; (which is useful for viewing the raw message) + (when (and maybe-view (mu4e~headers-view-this-message-p docid)) + (save-excursion (mu4e-view msg))) + ;; now, if this update was about *moving* a message, we don't show it + ;; anymore (of course, we cannot be sure if the message really no + ;; longer matches the query, but this seem a good heuristic. if it + ;; was only a flag-change, show the message with its updated flags. + (unless is-move + (save-excursion + (mu4e~headers-insert-header msg point))) + + ;; restore the mark, if any. See #2076. + (when (and markinfo (mu4e~headers-goto-docid docid)) + (mu4e-mark-at-point (car markinfo) (cdr markinfo))) + + (if (and initial-message-at-point + (mu4e~headers-goto-docid initial-message-at-point)) + (progn + (move-to-column initial-column) + (mu4e~headers-highlight initial-message-at-point)) + ;; attempt to highlight the corresponding line and make it visible + (mu4e~headers-highlight docid)) + (run-hooks 'mu4e-message-changed-hook)))))) + +(defun mu4e~headers-remove-handler (docid) + "Remove handler, will be called when a message with DOCID has +been removed from the database. This function will hide the removed +message from the current list of headers. If the message is not +present, don't do anything." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (mu4e~headers-remove-header docid t)) + ;; if we were viewing this message, close it now. + (when (and (mu4e~headers-view-this-message-p docid) + (buffer-live-p (mu4e-get-view-buffer))) + (let ((buf (mu4e-get-view-buffer))) + (mapc #'delete-window (get-buffer-window-list + buf nil t)) + (kill-buffer buf)))) + + + +;;; Performing queries (internal) +(defconst mu4e~search-message "Searching...") +(defconst mu4e~no-matches "No matching messages found") +(defconst mu4e~end-of-results "End of search results") + +(defvar mu4e--search-background nil + "Is this a background search? + If so, do not attempt to switch buffers. This variable is to be let-bound +to t before \"automatic\" searches.") + +(defun mu4e--search-execute (expr ignore-history) + "Search for query EXPR. + +Switch to the output buffer for the results. If IGNORE-HISTORY is +true, do *not* update the query history stack." + (let* ((buf (mu4e-get-headers-buffer nil t)) + (view-window mu4e~headers-view-win) + (inhibit-read-only t) + (rewritten-expr (funcall mu4e-query-rewrite-function expr)) + (maxnum (unless mu4e-search-full mu4e-search-results-limit))) + (with-current-buffer buf + ;; NOTE: this resets all buffer-local variables, including + ;; `mu4e~headers-view-win', which may have a live window if the + ;; headers buffer already exists when `mu4e-get-headers-buffer' + ;; is called. + (mu4e-headers-mode) + (setq mu4e~headers-view-win view-window) + (unless ignore-history + ;; save the old present query to the history list + (when mu4e--search-last-query + (mu4e--search-push-query mu4e--search-last-query 'past))) + (setq mu4e--search-last-query rewritten-expr) + (setq list-buffers-directory rewritten-expr) + (mu4e--modeline-update)) + + ;; when the buffer is already visible, select it; otherwise, + ;; switch to it. + (unless (get-buffer-window buf (if mu4e--search-background 0 nil)) + (mu4e-display-buffer buf t)) + (run-hook-with-args 'mu4e-search-hook expr) + (mu4e~headers-clear mu4e~search-message) + (setq mu4e~headers-search-start (float-time)) + (mu4e--server-find + rewritten-expr + mu4e-search-threads + mu4e-search-sort-field + mu4e-search-sort-direction + maxnum + mu4e-search-skip-duplicates + mu4e-search-include-related))) + +(defun mu4e~headers-benchmark-message (count) + "Get some report message for messaging search and rendering speed." + (if (and mu4e-headers-report-render-time + mu4e~headers-search-start + mu4e~headers-render-start + (> count 0)) + (let ((render-time-ms (* 1000(- (float-time) mu4e~headers-render-start))) + (search-time-ms (* 1000(- (float-time) mu4e~headers-search-start)))) + (format (concat + "; search: %0.1f ms (%0.2f ms/msg)" + "; render: %0.1f ms (%0.2f ms/msg)") + search-time-ms (/ search-time-ms count) + render-time-ms (/ render-time-ms count))) + "")) + +(defun mu4e~headers-found-handler (count) + "Create a one line description of the number of headers found +after the end of the search results." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (with-current-buffer (mu4e-get-headers-buffer) + (save-excursion + (goto-char (point-max)) + (let ((inhibit-read-only t) + (str (if (zerop count) mu4e~no-matches mu4e~end-of-results)) + (msg (format "Found %d matching message%s; %d hidden%s" + count (if (= 1 count) "" "s") + mu4e~headers-hidden + (mu4e~headers-benchmark-message count)))) + + (insert (propertize str 'face 'mu4e-system-face 'intangible t)) + (unless (zerop count) + (mu4e-message "%s" msg)))) + + ;; if we need to jump to some specific message, do so now + (goto-char (point-min)) + (when mu4e--search-msgid-target + (if (eq (current-buffer) (window-buffer)) + (mu4e-headers-goto-message-id mu4e--search-msgid-target) + (let* ((pos (mu4e-headers-goto-message-id + mu4e--search-msgid-target))) + (when pos + (set-window-point (get-buffer-window nil t) pos))))) + (when (and mu4e--search-view-target (mu4e-message-at-point 'noerror)) + ;; view the message at point when there is one. + (mu4e-headers-view-message)) + (setq mu4e--search-view-target nil + mu4e--search-msgid-target nil) + (when (mu4e~headers-docid-at-point) + (mu4e~headers-highlight (mu4e~headers-docid-at-point))) + ;; maybe enable thread folding + (when mu4e-search-threads + (mu4e-thread-mode)))) + ;; run-hooks + (run-hooks 'mu4e-headers-found-hook)) + + +;;; Marking + +(defmacro mu4e~headers-defun-mark-for (mark) + "Define a function mu4e~headers-mark-MARK." + (let ((funcname (intern (format "mu4e-headers-mark-for-%s" mark))) + (docstring (format "Mark header at point with %s." mark))) + `(progn + (defun ,funcname () ,docstring + (interactive) + (mu4e-headers-mark-and-next ',mark)) + (put ',funcname 'definition-name ',mark)))) + +(mu4e~headers-defun-mark-for refile) +(mu4e~headers-defun-mark-for something) +(mu4e~headers-defun-mark-for delete) +(mu4e~headers-defun-mark-for trash) +(mu4e~headers-defun-mark-for flag) +(mu4e~headers-defun-mark-for move) +(mu4e~headers-defun-mark-for read) +(mu4e~headers-defun-mark-for unflag) +(mu4e~headers-defun-mark-for untrash) +(mu4e~headers-defun-mark-for unmark) +(mu4e~headers-defun-mark-for unread) +(mu4e~headers-defun-mark-for action) + +(declare-function mu4e-view-pipe "mu4e-view") + +(defvar mu4e-headers-mode-map + (let ((map (make-sparse-keymap))) + + (define-key map "q" #'mu4e~headers-quit-buffer) + (define-key map "g" #'mu4e-search-rerun) ;; for compatibility + + + (define-key map "%" #'mu4e-headers-mark-pattern) + (define-key map "t" #'mu4e-headers-mark-subthread) + (define-key map "T" #'mu4e-headers-mark-thread) + + (define-key map "," #'mu4e-sexp-at-point) + (define-key map ";" #'mu4e-context-switch) + + ;; navigation between messages + (define-key map "p" #'mu4e-headers-prev) + (define-key map "n" #'mu4e-headers-next) + (define-key map (kbd "<M-up>") #'mu4e-headers-prev) + (define-key map (kbd "<M-down>") #'mu4e-headers-next) + + (define-key map (kbd "[") #'mu4e-headers-prev-unread) + (define-key map (kbd "]") #'mu4e-headers-next-unread) + + (define-key map (kbd "{") #'mu4e-headers-prev-thread) + (define-key map (kbd "}") #'mu4e-headers-next-thread) + + ;; change the number of headers + (define-key map (kbd "C-+") #'mu4e-headers-split-view-grow) + (define-key map (kbd "C--") #'mu4e-headers-split-view-shrink) + (define-key map (kbd "<C-kp-add>") 'mu4e-headers-split-view-grow) + (define-key map (kbd "<C-kp-subtract>") + #'mu4e-headers-split-view-shrink) + + ;; switching to view mode (if it's visible) + (define-key map "y" #'mu4e-select-other-view) + + ;; marking/unmarking ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (define-key map (kbd "<backspace>") #'mu4e-headers-mark-for-trash) + (define-key map (kbd "d") #'mu4e-headers-mark-for-trash) + (define-key map (kbd "<delete>") #'mu4e-headers-mark-for-delete) + (define-key map (kbd "<deletechar>") #'mu4e-headers-mark-for-delete) + (define-key map (kbd "D") #'mu4e-headers-mark-for-delete) + (define-key map (kbd "m") #'mu4e-headers-mark-for-move) + (define-key map (kbd "r") #'mu4e-headers-mark-for-refile) + + (define-key map (kbd "?") #'mu4e-headers-mark-for-unread) + (define-key map (kbd "!") #'mu4e-headers-mark-for-read) + (define-key map (kbd "A") #'mu4e-headers-mark-for-action) + + (define-key map (kbd "u") #'mu4e-headers-mark-for-unmark) + (define-key map (kbd "+") #'mu4e-headers-mark-for-flag) + (define-key map (kbd "-") #'mu4e-headers-mark-for-unflag) + (define-key map (kbd "=") #'mu4e-headers-mark-for-untrash) + (define-key map (kbd "&") #'mu4e-headers-mark-custom) + + (define-key map (kbd "*") + #'mu4e-headers-mark-for-something) + (define-key map (kbd "<kp-multiply>") + #'mu4e-headers-mark-for-something) + (define-key map (kbd "<insertchar>") + #'mu4e-headers-mark-for-something) + (define-key map (kbd "<insert>") + #'mu4e-headers-mark-for-something) + + (define-key map (kbd "#") #'mu4e-mark-resolve-deferred-marks) + + (define-key map "U" #'mu4e-mark-unmark-all) + (define-key map "x" #'mu4e-mark-execute-all) + + (define-key map "a" #'mu4e-headers-action) + + ;; message composition + + (define-key map (kbd "RET") #'mu4e-headers-view-message) + (define-key map [mouse-2] #'mu4e-headers-view-message) + + (define-key map "$" #'mu4e-show-log) + (define-key map "H" #'mu4e-display-manual) + + (define-key map "|" #'mu4e-view-pipe) + map) + "Keymap for mu4e's headers mode.") + +(easy-menu-define mu4e-headers-mode-menu + mu4e-headers-mode-map "Menu for mu4e's headers-mode." + (append + '("Headers" ;;:visible mu4e-headers-mode + "--" + ["Previous" mu4e-headers-prev + :help "Move to previous header"] + ["Next" mu4e-headers-prev + :help "Move to next header"] + "--" + ["Mark for move" mu4e-headers-mark-for-move + :help "Mark message for move" + ]) + mu4e--compose-menu-items + mu4e--search-menu-items + mu4e--context-menu-items + '( + "--" + ["Quit" mu4e~headers-quit-buffer + :help "Quit the headers"] + ))) + +;;; Headers-mode and mode-map + +(defun mu4e~header-line-format () + "Get the format for the header line." + (let ((uparrow (if mu4e-use-fancy-chars " ▲" " ^")) + (downarrow (if mu4e-use-fancy-chars " ▼" " V"))) + (cons + (make-string + (+ mu4e--mark-fringe-len (floor (fringe-columns 'left t))) ?\s) + (mapcar + (lambda (item) + (let* (;; with threading enabled, we're necessarily sorting by date. + (sort-field (if mu4e-search-threads + :date mu4e-search-sort-field)) + (field (car item)) (width (cdr item)) + (info (cdr (assoc field + (append mu4e-header-info + mu4e-header-info-custom)))) + (sortable (plist-get info :sortable)) + ;; if sortable, it is either t (when field is sortable itself) + ;; or a symbol (if another field is used for sorting) + (this-field (when sortable (if (booleanp sortable) + field + sortable))) + (help (plist-get info :help)) + ;; triangle to mark the sorted-by column + (arrow + (when (and sortable (eq this-field sort-field)) + (if (eq mu4e-search-sort-direction 'descending) + downarrow + uparrow))) + (name (concat (plist-get info :shortname) arrow)) + (map (make-sparse-keymap))) + (when sortable + (define-key map [header-line mouse-1] + (lambda (&optional e) + ;; getting the field, inspired by + ;; `tabulated-list-col-sort' + (interactive "e") + (let* ((obj (posn-object (event-start e))) + (field + (and obj + (get-text-property 0 'field (car obj))))) + ;; "t": if we're already sorted by field, the + ;; sort-order is changed + (mu4e-search-change-sorting field t))))) + (concat + (propertize + (if width + (truncate-string-to-width + name width 0 ?\s truncate-string-ellipsis) + name) + 'face (when arrow 'bold) + 'help-echo help + 'mouse-face (when sortable 'highlight) + 'keymap (when sortable map) + 'field field) " "))) + mu4e-headers-fields)))) + +(defun mu4e~headers-maybe-auto-update () + "Update the current headers buffer after indexing has brought +some changes, `mu4e-headers-auto-update' is non-nil and there is +no user-interaction ongoing." + (when (and mu4e-headers-auto-update ;; must be set + mu4e-index-update-status + (not (mu4e-get-view-buffer)) ;; not when viewing a message + (not (zerop (plist-get mu4e-index-update-status :updated))) + ;; NOTE: `mu4e-mark-marks-num' can return nil. Is that intended? + (zerop (or (mu4e-mark-marks-num) 0)) ;; non active marks + (not (active-minibuffer-window))) ;; no user input only + ;; rerun search if there's a live window with search results; + ;; otherwise we'd trigger a headers view from out of nowhere. + (when (and (buffer-live-p (mu4e-get-headers-buffer)) + (window-live-p (get-buffer-window (mu4e-get-headers-buffer) t))) + (let ((mu4e--search-background t)) + (mu4e-search-rerun))))) + +(defcustom mu4e-headers-eldoc-format "“%s” from %f on %d" + "Format for the `eldoc' string for the current message in the headers buffer. +The following specs are supported: +- %s: the message Subject +- %f: the message From +- %t: the message To +- %c: the message Cc +- %d: the message Date +- %p: the message priority +- %m: the maildir containing the message +- %F: the message’s flags +- %M: the Message-Id" + :type 'string + :group 'mu4e-headers) + +(defun mu4e-headers-eldoc-function (&rest _args) + (let ((msg (get-text-property (point) 'msg))) + (when msg + (format-spec + mu4e-headers-eldoc-format + `((?s . ,(mu4e-message-field msg :subject)) + (?f . ,(mu4e~headers-contact-str (mu4e-message-field msg :from))) + (?t . ,(mu4e~headers-contact-str (mu4e-message-field msg :to))) + (?c . ,(mu4e~headers-contact-str (mu4e-message-field msg :cc))) + (?d . ,(mu4e~headers-human-date msg)) + (?p . ,(mu4e-message-field msg :priority)) + (?m . ,(mu4e-message-field msg :maildir)) + (?F . ,(mu4e-message-field msg :flags)) + (?M . ,(mu4e-message-field msg :message-id))))))) + +(define-derived-mode mu4e-headers-mode special-mode + "mu4e:headers" + "Major mode for displaying mu4e search results. +\\{mu4e-headers-mode-map}." + (use-local-map mu4e-headers-mode-map) + (make-local-variable 'mu4e~headers-proc) + (make-local-variable 'mu4e~highlighted-docid) + (set (make-local-variable 'hl-line-face) 'mu4e-header-highlight-face) + + ;; Eldoc support + (when (and (featurep 'eldoc) mu4e-eldoc-support) + (if (boundp 'eldoc-documentation-functions) + ;; Emacs 28 or newer + (add-hook 'eldoc-documentation-functions + #'mu4e-headers-eldoc-function nil t) + ;; Emacs 27 or older + (add-function :before-until (local 'eldoc-documentation-function) + #'mu4e-headers-eldoc-function))) + + ;; support bookmarks. + (set (make-local-variable 'bookmark-make-record-function) + 'mu4e--make-bookmark-record) + ;; maybe update the current headers upon indexing changes + (add-hook 'mu4e-index-updated-hook #'mu4e~headers-maybe-auto-update) + (setq + truncate-lines t + buffer-undo-list t ;; don't record undo information + overwrite-mode nil + header-line-format (mu4e~header-line-format)) + + (mu4e--mark-initialize) ;; initialize the marking subsystem + (mu4e-context-minor-mode) + (mu4e-update-minor-mode) + (mu4e-search-minor-mode) + (mu4e-compose-minor-mode) + (hl-line-mode 1) + + (mu4e--modeline-register #'mu4e--search-modeline-item) + (mu4e--modeline-update)) + +;;; Highlighting + +(defvar mu4e~highlighted-docid nil + "The highlighted docid") + +(defun mu4e~headers-highlight (docid) + "Highlight the header with DOCID, or do nothing if it's not found. +Also, unhighlight any previously highlighted headers." + (with-current-buffer (mu4e-get-headers-buffer) + (save-excursion + ;; first, unhighlight the previously highlighted docid, if any + (when (and docid mu4e~highlighted-docid + (mu4e~headers-goto-docid mu4e~highlighted-docid)) + (hl-line-unhighlight)) + ;; now, highlight the new one + (when (mu4e~headers-goto-docid docid) + (hl-line-highlight))) + (setq mu4e~highlighted-docid docid))) + +;;; Misc 2 + +(defun mu4e~headers-select-window () + "When there is a visible window for the headers buffer, make sure +to select it. This is needed when adding new headers, otherwise +adding a lot of new headers looks really choppy." + (let ((win (get-buffer-window (mu4e-get-headers-buffer)))) + (when win (select-window win)))) + +(defun mu4e-headers-goto-message-id (msgid) + "Go to the next message with message-id MSGID. Return the +message plist, or nil if not found." + (mu4e-headers-find-if + (lambda (msg) + (let ((this-msgid (mu4e-message-field msg :message-id))) + (when (and this-msgid (string= msgid this-msgid)) + msg))))) + +;;; Marking 2 + +(defun mu4e~headers-mark (docid mark) + "(Visually) mark the header for DOCID with character MARK." + (with-current-buffer (mu4e-get-headers-buffer) + (let ((inhibit-read-only t) (oldpoint (point))) + (unless (mu4e~headers-goto-docid docid) + (mu4e-error "Cannot find message with docid %S" docid)) + ;; now, we're at the beginning of the header, looking at + ;; <docid>\004 + ;; (which is invisible). jump past that… + (unless (re-search-forward mu4e~headers-docid-post nil t) + (mu4e-error "Cannot find the `mu4e~headers-docid-post' separator")) + + ;; clear old marks, and add the new ones. + (let ((msg (get-text-property (point) 'msg))) + (delete-char mu4e--mark-fringe-len) + (insert (propertize + (format mu4e--mark-fringe-format mark) + 'face 'mu4e-header-marks-face + 'docid docid + 'msg msg))) + (goto-char oldpoint)))) + + +;;; Queries & searching + +;;; Search-based marking + +(defun mu4e-headers-for-each (func) + "Call FUNC for each header, moving point to the header. +FUNC receives one argument, the message s-expression for the +corresponding header." + (save-excursion + (goto-char (point-min)) + (while (search-forward mu4e~headers-docid-pre nil t) + ;; not really sure why we need to jump to bol; we do need to, otherwise we + ;; miss lines sometimes... + (let ((msg (get-text-property (line-beginning-position) 'msg))) + (when msg + (funcall func msg)))))) + + +(defun mu4e-headers-find-if (func &optional backward) + "Move to the header for which FUNC returns non-`nil'. +if BACKWARD is non-nil, search backwards. + + FUNC receives one argument, the message s-expression for the +corresponding header. If BACKWARD is non-`nil', search backwards. +Returns the new position, or `nil' if nothing was found. If you +want to exclude matches for the current message, you can use +`mu4e-headers-find-if-next'. + +Return the found position or nil if not found." + (let ((pos) + (search-func (if backward 'search-backward 'search-forward))) + (save-excursion + (while (and (null pos) + (funcall search-func mu4e~headers-docid-pre nil t)) + ;; not really sure why we need to jump to bol; we do need to, otherwise + ;; we miss lines sometimes... + (let ((msg (get-text-property (line-beginning-position) 'msg))) + (when (and msg (funcall func msg)) + (setq pos (point)))))) + (when pos + (goto-char pos)) + pos)) + +(defun mu4e-headers-find-if-next (func &optional backwards) + "Like `mu4e-headers-find-if', but do not match the current header. +Move to the next or (if BACKWARDS is non-`nil') header for which FUNC +returns non-`nil', starting from the current position." + (let ((pos)) + (save-excursion + (if backwards (beginning-of-line) (end-of-line)) + (setq pos (mu4e-headers-find-if func backwards))) + (when pos (goto-char pos)))) + +(defvar mu4e~headers-regexp-hist nil + "History list of regexps used.") + +(defun mu4e-headers-mark-for-each-if (markpair mark-pred &optional param) + "Mark all headers for which predicate function MARK-PRED returns +non-nil with MARKPAIR. MARK-PRED is function that receives two +arguments, MSG (the message at point) and PARAM (a user-specified +parameter). MARKPAIR is a cell (MARK . TARGET); see +`mu4e-mark-at-point' for details about marks." + (mu4e-headers-for-each + (lambda (msg) + (when (funcall mark-pred msg param) + (mu4e-mark-at-point (car markpair) (cdr markpair)))))) + +(defun mu4e-headers-mark-pattern () + "Ask user for a kind of mark (move, delete etc.), a field to +match and a regular expression to match with. Then, mark all +matching messages with that mark." + (interactive) + (let ((markpair (mu4e--mark-get-markpair "Mark matched messages with: " t)) + (field (mu4e-read-option "Field to match: " + '( ("subject" . :subject) + ("from" . :from) + ("to" . :to) + ("cc" . :cc) + ("bcc" . :bcc) + ("list" . :list)))) + (pattern (read-string + (mu4e-format "Regexp:") + nil 'mu4e~headers-regexp-hist))) + (mu4e-headers-mark-for-each-if + markpair + (lambda (msg _param) + (let* ((value (mu4e-msg-field msg field))) + (if (member field '(:to :from :cc :bcc :reply-to)) + (cl-find-if (lambda (contact) + (let ((name (mu4e-contact-name contact)) + (email (mu4e-contact-email contact))) + (or (and name (string-match pattern name)) + (and email (string-match pattern email))))) + value) + (string-match pattern (or value "")))))))) + +(defun mu4e-headers-mark-custom () + "Mark messages based on a user-provided predicate function." + (interactive) + (let* ((pred (mu4e-read-option "Match function: " + mu4e-headers-custom-markers)) + (param (when (cdr pred) (eval (cdr pred)))) + (markpair (mu4e--mark-get-markpair "Mark matched messages with: " t))) + (mu4e-headers-mark-for-each-if markpair (car pred) param))) + +(defun mu4e~headers-get-thread-info (msg what) + "Get WHAT (a symbol, either path or thread-id) for MSG." + (let* ((meta (or (mu4e-message-field msg :meta) + (mu4e-error "No thread info found"))) + (path (or (plist-get meta :path) + (mu4e-error "No threadpath found")))) + (cl-case what + (path path) + (thread-id + (save-match-data + ;; the thread id is the first segment of the thread path + (when (string-match "^\\([[:xdigit:]]+\\):?" path) + (match-string 1 path)))) + (otherwise (mu4e-error "Not supported"))))) + +(defun mu4e-headers-mark-thread-using-markpair (markpair &optional subthread) + "Mark the thread at point using the given markpair. If SUBTHREAD is +non-nil, marking is limited to the message at point and its +descendants." + (let* ((mark (car markpair)) + (allowed-marks (mapcar 'car mu4e-marks))) + (unless (memq mark allowed-marks) + (mu4e-error "The mark (%s) has to be one of: %s" + mark allowed-marks))) + ;; note: the thread id is shared by all messages in a thread + (let* ((msg (mu4e-message-at-point)) + (thread-id (mu4e~headers-get-thread-info msg 'thread-id)) + (path (mu4e~headers-get-thread-info msg 'path)) + ;; the thread path may have a ':z' suffix for sorting; + ;; remove it for subthread matching. + (match-path (replace-regexp-in-string ":z$" "" path)) + (last-marked-point)) + (mu4e-headers-for-each + (lambda (cur-msg) + (let ((cur-thread-id (mu4e~headers-get-thread-info cur-msg 'thread-id)) + (cur-thread-path (mu4e~headers-get-thread-info cur-msg 'path))) + (if subthread + ;; subthread matching; mymsg's thread path should have path as its + ;; prefix + (when (string-match (concat "^" match-path) cur-thread-path) + (mu4e-mark-at-point (car markpair) (cdr markpair)) + (setq last-marked-point (point))) + ;; nope; not looking for the subthread; looking for the whole thread + (when (string= thread-id cur-thread-id) + (mu4e-mark-at-point (car markpair) (cdr markpair)) + (setq last-marked-point (point))))))) + (when last-marked-point + (goto-char last-marked-point) + (mu4e-headers-next)))) + +(defun mu4e-headers-mark-thread (&optional subthread markpair) + "Like `mu4e-headers-mark-thread-using-markpair' but prompt for the markpair." + (interactive + (let* ((subthread current-prefix-arg)) + (list current-prefix-arg + ;; FIXME: e.g., for refiling we should evaluate this + ;; for each line separately + (mu4e--mark-get-markpair + (if subthread "Mark subthread with: " "Mark whole thread with: ") + t)))) + (mu4e-headers-mark-thread-using-markpair markpair subthread)) + +(defun mu4e-headers-mark-subthread (&optional markpair) + "Like `mu4e-mark-thread', but only for a sub-thread." + (interactive) + (if markpair (mu4e-headers-mark-thread t markpair) + (let ((current-prefix-arg t)) + (call-interactively 'mu4e-headers-mark-thread)))) + + + +(defun mu4e-headers-view-message () + "View message at point." + (interactive) + (unless (eq major-mode 'mu4e-headers-mode) + (mu4e-error "Must be in mu4e-headers-mode (%S)" major-mode)) + (let* ((msg (mu4e-message-at-point)) + (path (mu4e-message-field msg :path)) + (_exists (or (file-readable-p path) + (mu4e-warn "No message at %s" path))) + (docid (or (mu4e-message-field msg :docid) + (mu4e-warn "No message at point"))) + (mark-as-read + (if (functionp mu4e-view-auto-mark-as-read) + (funcall mu4e-view-auto-mark-as-read msg) + mu4e-view-auto-mark-as-read))) + (when-let ((buf (mu4e-get-view-buffer (current-buffer) nil))) + (with-current-buffer buf + (mu4e-loading-mode 1))) + (mu4e--server-view docid mark-as-read))) + +(defvar-local mu4e-headers-open-after-move t + "If set to non-nil, open message after `mu4e-headers-next' and +`mu4e-headers-prev' if pointing at a message after the move +and there is a live message view. + +This variable is for let-binding when scripting.") + +(defun mu4e~headers-move (lines) + "Move point LINES lines. +Move forward if LINES is positive or backwards if LINES is +negative. If this succeeds, return the new docid. Otherwise, +return nil. + +If pointing at a message after the move and there is a +view-window, open the message unless +`mu4e-headers-open-after-move' is non-nil." + (cl-assert (eq major-mode 'mu4e-headers-mode)) + (when (ignore-errors + (let (line-move-visual) + (line-move lines) + t)) + (let* ((docid (mu4e~headers-docid-at-point)) + (folded (and docid (mu4e-thread-message-folded-p)))) + (if folded + (mu4e~headers-move (if (< lines 0) -1 1)) ;; skip folded + (when docid + ;; Skip invisible text at BOL possibly hidden by + ;; the end of another invisible overlay covering + ;; previous EOL. + (move-to-column 2) + ;; update all windows showing the headers buffer + (walk-windows + (lambda (win) + (when (eq (window-buffer win) + (mu4e-get-headers-buffer (buffer-name))) + (set-window-point win (point)))) + nil t) + ;; If the assigned (and buffer-local) `mu4e~headers-view-win' + ;; is not live then that is indicates the user does not want + ;; to pop up the view when they navigate in the headers + ;; buffer. + (when (and mu4e-headers-open-after-move + (window-live-p mu4e~headers-view-win)) + (mu4e-headers-view-message)) + ;; attempt to highlight the new line, display the message + (mu4e~headers-highlight docid) + docid))))) + +(defun mu4e-headers-next (&optional n) + "Move point to the next message header. +If this succeeds, return the new docid. Otherwise, return nil. +Optionally, takes an integer N (prefix argument), to the Nth next +header. + +If pointing at a message after the move and there is a +view-window, open the message unless +`mu4e-headers-open-after-move' is non-nil." + (interactive "P") + (mu4e~headers-move (or n 1))) + +(defun mu4e-headers-prev (&optional n) + "Move point to the previous message header. +If this succeeds, return the new docid. Otherwise, return nil. +Optionally, takes an integer N (prefix argument), to the Nth +previous header. + +If pointing at a message after the move and there is a +view-window, open the message unless +`mu4e-headers-open-after-move' is non-nil." + (interactive "P") + (mu4e~headers-move (- (or n 1)))) + +(defun mu4e~headers-prev-or-next-unread (backwards) + "Move point to the next message that is unread (and +untrashed). If BACKWARDS is non-`nil', move backwards." + (interactive "P") + (or (mu4e-headers-find-if-next + (lambda (msg) + (let ((flags (mu4e-message-field msg :flags))) + (and (member 'unread flags) (not (member 'trashed flags))))) + backwards) + (mu4e-message (format "No %s unread message found" + (if backwards "previous" "next"))))) + +(defun mu4e-headers-prev-unread () + "Move point to the previous message that is unread (and +untrashed)." + (interactive) + (mu4e~headers-prev-or-next-unread t)) + +(defun mu4e-headers-next-unread () + "Move point to the next message that is unread (and +untrashed)." + (interactive) + (mu4e~headers-prev-or-next-unread nil)) + +(defun mu4e~headers-thread-root-p (&optional msg) + "Is MSG at the root of a thread? +If MSG is nil, use message at point." + (when-let* ((msg (or msg (get-text-property (point) 'msg))) + (meta (mu4e-message-field msg :meta))) + (let* ((orphan (plist-get meta :orphan)) + (first-child (plist-get meta :first-child)) + (root (plist-get meta :root))) + (or root (and orphan first-child))))) + +(defun mu4e~headers-prev-or-next-thread (backwards) + "Move point to the top of the next thread. +If BACKWARDS is non-`nil', move backwards." + (when (mu4e-headers-find-if-next #'mu4e~headers-thread-root-p backwards) + (point))) + +(defun mu4e-headers-prev-thread () + "Move point to the previous thread." + (interactive) (mu4e~headers-prev-or-next-thread t)) + +(defun mu4e-headers-next-thread () + "Move point to the previous thread." + (interactive) (mu4e~headers-prev-or-next-thread nil)) + +(defun mu4e-headers-split-view-grow (&optional n) + "In split-view, grow the headers window. +In horizontal split-view, increase the number of lines shown by N. +In vertical split-view, increase the number of columns shown by N. +If N is negative shrink the headers window. When not in split-view +do nothing." + (interactive "P") + (let ((n (or n 1)) + (hwin (get-buffer-window (mu4e-get-headers-buffer)))) + (when (and (buffer-live-p (mu4e-get-view-buffer)) (window-live-p hwin)) + (let ((n (or n 1))) + (cl-case mu4e-split-view + ;; emacs has weird ideas about what horizontal, vertical means... + (horizontal + (window-resize hwin n nil) + (cl-incf mu4e-headers-visible-lines n)) + (vertical + (window-resize hwin n t) + (cl-incf mu4e-headers-visible-columns n))))))) + +(defun mu4e-headers-split-view-shrink (&optional n) + "In split-view, shrink the headers window. +In horizontal split-view, decrease the number of lines shown by N. +In vertical split-view, decrease the number of columns shown by N. +If N is negative grow the headers window. When not in split-view +do nothing." + (interactive "P") + (mu4e-headers-split-view-grow (- (or n 1)))) + +(defun mu4e-headers-action (&optional actionfunc) + "Ask user what to do with message-at-point, then do it. +The actions are specified in `mu4e-headers-actions'. Optionally, +pass ACTIONFUNC, which is a function that takes a msg-plist +argument." + (interactive) + (let ((msg (mu4e-message-at-point)) + (afunc (or actionfunc + (mu4e-read-option "Action: " mu4e-headers-actions)))) + (funcall afunc msg))) + +(defun mu4e-headers-mark-and-next (mark) + "Set mark MARK on the message at point or on all messages in the +region if there is a region, then move to the next message." + (interactive) + (when (mu4e-thread-message-folded-p) + (mu4e-warn "Cannot mark folded messages")) + (mu4e-mark-set mark) + (when mu4e-headers-advance-after-mark (mu4e-headers-next))) + +(defun mu4e~headers-quit-buffer () + "Quit the mu4e-headers buffer and go back to the main view." + (interactive) + (mu4e-mark-handle-when-leaving) + (quit-window t) + ;; clear the decks before going to the main-view + (mu4e--query-items-refresh 'reset-baseline) + (mu4e--main-view)) + + +;;; Loading messages +;; + + +(defvar-local mu4e--loading-overlay-bg nil + "Internal variable that holds the loading overlay for the background.") + +(defvar-local mu4e--loading-overlay-text nil + "Internal variable that holds the loading overlay for the text.") + +(define-minor-mode mu4e-loading-mode + "Minor mode for buffers awaiting data from mu" + :init-value nil :lighter " Loading" :keymap nil + (if mu4e-loading-mode + (progn + (when mu4e-dim-when-loading + (setq mu4e--loading-overlay-bg + (let ((overlay (make-overlay (point-min) (point-max)))) + (overlay-put overlay 'face + `(:foreground "gray22" :background + ,(face-attribute 'default + :background))) + (overlay-put overlay 'priority 9998) + overlay)) + (setq mu4e--loading-overlay-text + (let ((overlay (make-overlay (point-min) (point-min)))) + (overlay-put overlay 'priority 9999) + (overlay-put overlay 'before-string + (propertize "Loading…\n" + 'face 'mu4e-header-title-face)) + overlay)))) + (when mu4e--loading-overlay-bg + (delete-overlay mu4e--loading-overlay-bg)) + (when mu4e--loading-overlay-text + (delete-overlay mu4e--loading-overlay-text)))) + +(provide 'mu4e-headers) +;;; mu4e-headers.el ends here diff --git a/mu4e/mu4e-helpers.el b/mu4e/mu4e-helpers.el new file mode 100644 index 0000000..e2718bb --- /dev/null +++ b/mu4e/mu4e-helpers.el @@ -0,0 +1,604 @@ +;;; mu4e-helpers.el --- Helper functions -*- lexical-binding: t -*- + +;; Copyright (C) 2022-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Helper functions used in the mu4e. This is slowly usurp all the code from +;; mu4e-utils.el that does not depend on other parts of mu4e. + +;;; Code: + +(require 'seq) +(require 'ido) +(require 'cl-lib) +(require 'bookmark) + +(require 'mu4e-window) +(require 'mu4e-config) + +;;; Customization + +(defcustom mu4e-debug nil + "When set to non-nil, log debug information to the mu4e log buffer." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-completing-read-function #'ido-completing-read + "Function to be used to receive user-input during completion. + +Suggested possible values are: + * `completing-read': emacs built-in completion method + * `ido-completing-read': dynamic completion within the minibuffer. + +The function is used in two contexts - +1) directly - for instance in when listing _other_ maildirs + in `mu4e-ask-maildir' +2) if `mu4e-read-option-use-builtin' is nil, it is used + as part of `mu4e-read-option' in many places. + +Set it to `completing-read' when you want to use completion +frameworks such as Helm, Ivy or Vertico. In that case, you +might want to add something like the following in your configuration. + + (setq mu4e-read-option-use-builtin nil + mu4e-completing-read-function \\='completing-read) +." + :type 'function + :options '(completing-read ido-completing-read) + :group 'mu4e) + +(defcustom mu4e-read-option-use-builtin t + "Whether to use mu4e's traditional completion for +`mu4e-read-option'. + +If nil, use the value of `mu4e-completing-read-function', integrated +into mu4e. + +Many of the third-party completion frameworks - such as Helm, Ivy +and Vertico - influence `completion-read', so to have mu4e follow +your overall settings, try the equivalent of + + (setq mu4e-read-option-use-builtin nil + mu4e-completing-read-function \\='completing-read) + +Tastes differ, but without any such frameworks, the unaugmented +Emacs `completing-read' is rather Spartan." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-use-fancy-chars nil + "When set, allow fancy (Unicode) characters for marks/threads. +You can customize the exact fancy characters used with +`mu4e-marks' and various `mu4e-headers-..-mark' and +`mu4e-headers..-prefix' variables." + :type 'boolean + :group 'mu4e) + +;; maybe move the next ones... but they're convenient +;; here because they're needed in multiple buffers. + +(defcustom mu4e-view-auto-mark-as-read t + "Automatically mark messages as read when you read them. +This is the default behavior, but can be turned off, for example +when using a read-only file-system. + +This can also be set to a function; if so, receives a message +plist which should evaluate to nil if the message should *not* be +marked as read-only, or non-nil otherwise." + :type '(choice + boolean + function) + :group 'mu4e-view) + + + +(defun mu4e-select-other-view () + "Switch between headers view and message view." + (interactive) + (let* ((other-buf + (cond + ((mu4e-current-buffer-type-p 'view) + (mu4e-get-headers-buffer)) + ((mu4e-current-buffer-type-p 'headers) + (mu4e-get-view-buffer)) + (t (mu4e-error + "This window is neither the headers nor the view window.")))) + (other-win (and other-buf (get-buffer-window other-buf)))) + (if (window-live-p other-win) + (select-window other-win) + (mu4e-message "No window to switch to")))) + + + +;;; Messages, warnings and errors +(defun mu4e-format (frm &rest args) + "Create [mu4e]-prefixed string based on format FRM and ARGS." + (concat + "[" (propertize "mu4e" 'face 'mu4e-title-face) "] " + (apply 'format frm + (mapcar (lambda (x) + (if (stringp x) + (decode-coding-string x 'utf-8) + x)) + args)))) + +(defun mu4e-message (frm &rest args) + "Display FRM with ARGS like `message' in mu4e style. +If we're waiting for user-input or if there's some message in the +echo area, don't show anything." + (unless (or (active-minibuffer-window)) + (message "%s" (apply 'mu4e-format frm args)))) + +(declare-function mu4e~loading-close "mu4e-headers") + +(defun mu4e-error (frm &rest args) + "Display an error with FRM and ARGS like `mu4e-message'. + +Create [mu4e]-prefixed error based on format FRM and ARGS. Does a +local-exit and does not return, and raises a +debuggable (backtrace) error." + (mu4e-log 'error (apply 'mu4e-format frm args)) + (error "%s" (apply 'mu4e-format frm args))) + +(defun mu4e-warn (frm &rest args) + "Create [mu4e]-prefixed warning based on format FRM and ARGS. +Does a local-exit and does not return." + (mu4e-log 'error (apply 'mu4e-format frm args)) + (user-error "%s" (apply 'mu4e-format frm args))) + +;;; Reading user input + +(defun mu4e--plist-get (lst prop) + "Get PROP from plist LST and raise an error if not present." + (or (plist-get lst prop) + (if (plist-member lst prop) + nil + (mu4e-error "Missing property %s in %s" prop lst)))) + +(defun mu4e--matching-choice (choices kar) + "Does KAR match any of the CHOICES? + +KAR is a character and CHOICES is an alist as described in +`mu4e--read-choice-builtin'. + +First try an exact match, but if there isn't, try +case-insensitive. + +Return the cdr (value) of the matching cell, if any." + (let* ((match) (match-ci)) + (catch 'found + (seq-do + (lambda (choice) + ;; first try an exact match + (let ((case-fold-search nil)) + (if (char-equal kar (caadr choice)) + (progn + (setq match choice) + (throw 'found choice)) ;; found it - quit. + ;; perhaps case-insensitive? + (let ((case-fold-search t)) + (when (and (not match-ci) (char-equal kar (caadr choice))) + (setq match-ci choice)))))) + choices)) + (if match (cdadr match) + (when match-ci (cdadr match-ci))))) + +(defun mu4e--read-choice-completing-read (prompt choices) + "Read and return one of CHOICES, prompting for PROMPT. + +PROMPT describes a multiple-choice question to the user. CHOICES +is an alist of the form + ( ( <display-string> ( <shortcut> . <value> )) + ... ) +Any input that is not one of CHOICES is ignored. This is mu4e's +version of `read-char-choice' which becomes case-insensitive +after trying an exact match. + +Return the matching choice value (cdr of the cell)." + (let* ((metadata `(metadata + (display-sort-function . ,#'identity) + (cycle-sort-function . ,#'identity))) + (quick-result) + (result + (minibuffer-with-setup-hook + (lambda () + (add-hook 'post-command-hook + (lambda () + ;; Exit directly if a quick key is pressed + (let ((prefix (minibuffer-contents-no-properties))) + (unless (string-empty-p prefix) + (setq quick-result + (mu4e--matching-choice + choices (string-to-char prefix))) + (when quick-result + (exit-minibuffer))))) + -1 'local)) + (funcall mu4e-completing-read-function + prompt + ;; Use function with metadata to disable sorting. + (lambda (input predicate action) + (if (eq action 'metadata) + metadata + (complete-with-action action choices input predicate))) + ;; Require confirmation, if the input does not match a suggestion + nil t nil nil nil)))) + (or quick-result + (cdadr (assoc result choices))))) + +(defun mu4e--read-choice-builtin (prompt choices) + "Read and return one of CHOICES, prompting for PROMPT. + +PROMPT describes a multiple-choice question to the user. CHOICES +is an alist of the fiorm + ( ( <display-string> ( <shortcut> . <value> )) + ... ) +Any input that is not one of CHOICES is ignored. This is mu4e's +version of `read-char-choice' which becomes case-insensitive +after trying an exact match. + +Return the matching choice value (cdr of the cell)." + (let ((chosen) (inhibit-quit nil) + (prompt (format "%s%s" + (mu4e-format prompt) + (mapconcat #'car choices ", ")))) + (while (not chosen) + (message nil) ;; this seems needed... + (when-let ((kar (read-char-exclusive prompt))) + (when (eq kar ?\e) (keyboard-quit)) ;; `read-char-exclusive' is a C + ;; function and doesn't check for + ;; `keyboard-quit', there we need to + ;; check if ESC is pressed + (setq chosen (mu4e--matching-choice choices kar)))) + chosen)) + +(defun mu4e-read-option (prompt options) + "Ask user for an option from a list on the input area. + +PROMPT describes a multiple-choice question to the user. OPTIONS +describe the options, and is a list of cells describing +particular options. Cells have the following structure: + + (OPTION . RESULT) + +where OPTIONS is a non-empty string describing the option. The +first character of OPTION is used as the shortcut, and obviously +all shortcuts must be different, so you can prefix the string +with an uniquifying character. + +The options are provided as a list for the user to choose from; +user can then choose by typing CHAR. Example: + (mu4e-read-option \"Choose an animal: \" + \\='((\"Monkey\" . monkey) (\"Gnu\" . gnu) (\"xMoose\" . moose))) + +User now will be presented with a list: \"Choose an animal: + [M]onkey, [G]nu, [x]Moose\". + +If optional character KEY is provied, use that instead of asking +the user. + +Function returns the value (cdr) of the matching cell." + (let* ((choices ;; ((<display> ( <key> . <value> ) ...) + (seq-map + (lambda (option) + (list + (concat ;; <display> + "[" (propertize (substring (car option) 0 1) + 'face 'mu4e-highlight-face) + "]" + (substring (car option) 1)) + (cons + (string-to-char (car option)) ;; <key> + (cdr option)))) ;; <value> + options)) + (response (funcall + (if mu4e-read-option-use-builtin + #'mu4e--read-choice-builtin + #'mu4e--read-choice-completing-read) + prompt choices))) + (or response + (mu4e-warn "invalid input")))) + +(defun mu4e-filter-single-key (lst) + "Return a list consisting of LST items with a `characterp' :key prop." + ;; This works for bookmarks and maildirs. + (seq-filter (lambda (item) + (characterp (plist-get item :key))) + lst)) + + +;;; Logging / debugging + +(defconst mu4e--log-max-size 1000000 + "Max number of characters to keep around in the log buffer.") +(defconst mu4e--log-buffer-name "*mu4e-log*" + "Name of the logging buffer.") + +(defun mu4e--get-log-buffer () + "Fetch (and maybe create) the log buffer." + (unless (get-buffer mu4e--log-buffer-name) + (with-current-buffer (get-buffer-create mu4e--log-buffer-name) + (view-mode) + (when (fboundp 'so-long-mode) + (unless (eq major-mode 'so-long-mode) + (eval '(so-long-mode)))) + (setq buffer-undo-list t))) + mu4e--log-buffer-name) + +(defun mu4e-log (type frm &rest args) + "Log a message of TYPE with format-string FRM and ARGS. +Use the mu4e log buffer for this. If the variable mu4e-debug is +non-nil. Type is a symbol, either `to-server', `from-server' or +`misc'. + +This function is meant for debugging." + (when mu4e-debug + (with-current-buffer (mu4e--get-log-buffer) + (let* ((inhibit-read-only t) + (tstamp (propertize (format-time-string "%Y-%m-%d %T.%3N" + (current-time)) + 'face 'font-lock-string-face)) + (msg-face + (pcase type + ('from-server 'font-lock-type-face) + ('to-server 'font-lock-function-name-face) + ('misc 'font-lock-variable-name-face) + ('error 'font-lock-warning-face) + (_ (mu4e-error "Unsupported log type")))) + (msg (propertize (apply 'format frm args) 'face msg-face))) + (save-excursion + (goto-char (point-max)) + (insert tstamp + (pcase type + ('from-server " <- ") + ('to-server " -> ") + ('error " !! ") + (_ " ")) + msg "\n") + ;; if `mu4e-log-max-lines is specified and exceeded, clearest the + ;; oldest lines + (when (> (buffer-size) mu4e--log-max-size) + (goto-char (- (buffer-size) mu4e--log-max-size)) + (beginning-of-line) + (delete-region (point-min) (point)))))))) + +(defun mu4e-toggle-logging () + "Toggle `mu4e-debug'. +In debug-mode, mu4e logs some of its internal workings to a +log-buffer. See `mu4e-show-log'." + (interactive) + (mu4e-log 'misc "logging disabled") + (setq mu4e-debug (not mu4e-debug)) + (mu4e-message "debug logging has been %s" + (if mu4e-debug "enabled" "disabled")) + (mu4e-log 'misc "logging enabled")) + +(defun mu4e-show-log () + "Visit the mu4e debug log." + (interactive) + (unless mu4e-debug (mu4e-toggle-logging)) + (let ((buf (get-buffer mu4e--log-buffer-name))) + (unless (buffer-live-p buf) + (mu4e-warn "No debug log available")) + (display-buffer buf))) + + + +;;; Flags +;; Converting flags->string and vice-versa + +(defun mu4e-flags-to-string (flags) + "Convert a list of Maildir[1] FLAGS into a string. + +See `mu4e-string-to-flags'. \[1\]: +http://cr.yp.to/proto/maildir.html." + (seq-sort + '< + (seq-mapcat + (lambda (flag) + (pcase flag + (`draft "D") + (`flagged "F") + (`new "N") + (`passed "P") + (`replied "R") + (`seen "S") + (`trashed "T") + (`attach "a") + (`encrypted "x") + (`signed "s") + (`unread "u") + (_ ""))) + (seq-uniq flags) 'string))) + +(defun mu4e-string-to-flags (str) + "Convert a STR with Maildir[1] flags into a list of flags. + +See `mu4e-string-to-flags'. \[1\]: +http://cr.yp.to/proto/maildir.html." + (seq-uniq + (seq-filter + 'identity + (seq-mapcat + (lambda (kar) + (list + (pcase kar + ('?D 'draft) + ('?F 'flagged) + ('?P 'passed) + ('?R 'replied) + ('?S 'seen) + ('?T 'trashed) + (_ nil)))) + str)))) + + +;;; Misc +(defun mu4e-copy-thing-at-point () + "Copy e-mail address or URL at point to the kill ring. +If there is not e-mail address at point, do nothing." + (interactive) + (let* ((thing (and (thing-at-point 'email) + (string-trim (thing-at-point 'email 'no-props) "<" ">"))) + (thing (or thing (get-text-property (point) 'shr-url))) + (thing (or thing (thing-at-point 'url 'no-props)))) + (when thing + (kill-new thing) + (mu4e-message "Copied '%s' to kill-ring" thing)))) + +(defun mu4e-display-size (size) + "Get a human-friendly string representation of SIZE (in bytes)." + (cond + ((>= size 1000000) + (format "%2.1fM" (/ size 1000000.0))) + ((and (>= size 1000) (< size 1000000)) + (format "%2.1fK" (/ size 1000.0))) + ((< size 1000) + (format "%d" size)) + (t "?"))) + +(defun mu4e-split-ranges-to-numbers (str n) + "Convert STR containing attachment numbers into a list of numbers. + +STR is a string; N is the highest possible number in the list. +This includes expanding e.g. 3-5 into 3,4,5. If the letter +\"a\" ('all')) is given, that is expanded to a list with numbers +[1..n]." + (let ((str-split (split-string str)) + beg end list) + (dolist (elem str-split list) + ;; special number "a" converts into all attachments 1-N. + (when (equal elem "a") + (setq elem (concat "1-" (int-to-string n)))) + (if (string-match "\\([0-9]+\\)-\\([0-9]+\\)" elem) + ;; we have found a range A-B, which needs converting + ;; into the numbers A, A+1, A+2, ... B. + (progn + (setq beg (string-to-number (match-string 1 elem)) + end (string-to-number (match-string 2 elem))) + (while (<= beg end) + (cl-pushnew beg list :test 'equal) + (setq beg (1+ beg)))) + ;; else just a number + (cl-pushnew (string-to-number elem) list :test 'equal))) + ;; Check that all numbers are valid. + (mapc + (lambda (x) + (cond + ((> x n) + (mu4e-warn "Attachment %d bigger than maximum (%d)" x n)) + ((< x 1) + (mu4e-warn "Attachment number must be greater than 0 (%d)" x)))) + list))) + +(defun mu4e-make-temp-file (ext) + "Create a self-destructing temporary file with extension EXT. +The file will self-destruct in a short while, enough to open it +in an external program." + (let ((tmpfile (make-temp-file "mu4e-" nil (concat "." ext)))) + (run-at-time "30 sec" nil + (lambda () (ignore-errors (delete-file tmpfile)))) + tmpfile)) + +(defun mu4e-display-manual () + "Display the mu4e manual page for the current mode. +Or go to the top level if there is none." + (interactive) + (info (pcase major-mode + ('mu4e-main-mode "(mu4e)Main view") + ('mu4e-headers-mode "(mu4e)Headers view") + ('mu4e-view-mode "(mu4e)Message view") + (_ "mu4e")))) + + +;;; bookmarks +(defun mu4e--make-bookmark-record () + "Create a bookmark for the message at point." + (let* ((msg (mu4e-message-at-point)) + (subject (or (plist-get msg :subject) "No subject")) + (date (plist-get msg :date)) + (date (if date (format-time-string "%F: " date) "")) + (title (format "%s%s" date subject)) + (msgid (or (plist-get msg :message-id) + (mu4e-error + "Cannot bookmark message without message-id")))) + `(,title + ,@(bookmark-make-record-default 'no-file 'no-context) + (message-id . ,msgid) + (handler . mu4e--jump-to-bookmark)))) + +(declare-function mu4e-view-message-with-message-id "mu4e-view") +(declare-function mu4e-message-at-point "mu4e-message") + +(defun mu4e--jump-to-bookmark (bookmark) + "View the message referred to by BOOKMARK." + (when-let ((msgid (bookmark-prop-get bookmark 'message-id))) + (mu4e-view-message-with-message-id msgid))) + + ;;; Macros + +(defmacro mu4e-setq-if-nil (var val) + "Set VAR to VAL if VAR is nil." + `(unless ,var (setq ,var ,val))) + + + +;;; Misc +(defun mu4e-join-paths (directory &rest components) + "Append COMPONENTS to DIRECTORY and return the resulting string. + +This is mu4e's version of Emacs 28's `file-name-concat' with the +difference it also handles slashes at the beginning of +COMPONENTS." + (replace-regexp-in-string + "//+" "/" + (mapconcat (lambda (part) (if (stringp part) part "")) + (cons directory components) "/"))) + +(defun mu4e-string-replace (from-string to-string in-string) + "Replace FROM-STRING with TO-STRING in IN-STRING each time it occurs. +Mu4e version of emacs 28's string-replace." + (replace-regexp-in-string (regexp-quote from-string) + to-string in-string nil 'literal)) + +(defun mu4e-plistp (object) + "Non-nil if and only if OBJECT is a valid plist. + +This is mu4e's version of Emacs 29's `plistp'." + (let ((len (proper-list-p object))) + (and len (zerop (% len 2))))) + +(defun mu4e-key-description (cmd) + "Get the textual form of current binding to interactive function CMD. +If it is unbound, return nil. If there are multiple bindings, +return the shortest. + +Roughly does what `substitute-command-keys' does, but picks +shorter keys in some cases where there are multiple bindings." + ;; not a perfect heuristic: e.g. '<up>' is longer that 'C-p' + (car-safe + (seq-sort (lambda (b1 b2) + (< (length b1) (length b2))) + (seq-map #'key-description + (where-is-internal cmd))))) + +(provide 'mu4e-helpers) +;;; mu4e-helpers.el ends here diff --git a/mu4e/mu4e-icalendar.el b/mu4e/mu4e-icalendar.el new file mode 100644 index 0000000..30feb51 --- /dev/null +++ b/mu4e/mu4e-icalendar.el @@ -0,0 +1,210 @@ +;;; mu4e-icalendar.el --- iCalendar & diary integration -*- lexical-binding: t; -*- + +;; Copyright (C) 2019-2023 Christophe Troestler + +;; Author: Christophe Troestler <Christophe.Troestler@umons.ac.be> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Keywords: email icalendar +;; Version: 0.0 + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; To install: +;; (require 'mu4e-icalendar) +;; (gnus-icalendar-setup) +;; Optional: +;; (setq mu4e-icalendar-trash-after-reply t) + +;; By default, the original message is not cited. However, if you +;; would like to reply to it, the citation is in the kill-ring (paste +;; it with `yank'). + +;; To add the event to a diary file of your choice: +;; (setq mu4e-icalendar-diary-file "/path/to/your/diary") +;; If the file specified is not your main diary file, add +;; #include "/path/to/your/diary" +;; to you main diary file to display the events. + +;; To enable optional iCalendar->Org sync functionality +;; NOTE: both the capture file and the headline(s) inside must already exist +;; (require 'org-agenda) +;; (setq gnus-icalendar-org-capture-file "~/org/notes.org") +;; (setq gnus-icalendar-org-capture-headline '("Calendar")) +;; (gnus-icalendar-org-setup) + +;;; Code: + +(require 'gnus-icalendar) +(require 'cl-lib) + +(require 'mu4e-mark) +(require 'mu4e-helpers) +(require 'mu4e-contacts) +(require 'mu4e-headers) +(require 'mu4e-obsolete) + + +;;; Configuration +;;;; Calendar + +(defgroup mu4e-icalendar nil + "Icalendar related settings." + :group 'mu4e) + +(defcustom mu4e-icalendar-trash-after-reply nil + "If non-nil, trash the icalendar invitation after replying." + :type 'boolean + :group 'mu4e-icalendar) + +(defcustom mu4e-icalendar-diary-file nil + "If non-nil, the file in which to add events upon reply." + :type '(choice (const :tag "Do not insert a diary entry" nil) + (string :tag "Insert a diary entry in this file")) + :group 'mu4e-icalendar) + + +(defun mu4e--icalendar-has-email (email list) + "Check that EMAIL is in LIST." + (let ((email (downcase email))) + (cl-find-if (lambda (c) (let ((e (mu4e-contact-email c))) + (and (stringp e) (string= email (downcase e))))) + list))) + +(declare-function mu4e--view-mode-p "mu4e-view") +(defun mu4e--icalendar-reply (orig data) + "Wrapper for using either `mu4e-icalender-reply' or the ORIG function." + (funcall (if (mu4e--view-mode-p) #'mu4e-icalendar-reply orig) data)) + +(advice-add #'gnus-icalendar-reply :around #'mu4e--icalendar-reply) +;;(advice-remove #'gnus-icalendar-reply #'mu4e--icalendar-reply) + +(defun mu4e-icalendar-reply (data) + "Reply to the text/calendar event present in DATA." + ;; Based on `gnus-icalendar-reply'. + (let* ((handle (car data)) + (status (cadr data)) + (event (caddr data)) + (gnus-icalendar-additional-identities + (mu4e-personal-addresses 'no-regexp)) + (reply (gnus-icalendar-with-decoded-handle + handle + (gnus-icalendar-event-reply-from-buffer + (current-buffer) status (gnus-icalendar-identities)))) + (msg (mu4e-message-at-point 'noerror)) + (charset (cdr (assoc 'charset (mm-handle-type handle))))) + (when reply + (cl-labels + ((fold-icalendar-buffer + () + (goto-char (point-min)) + (while (re-search-forward "^\\(.\\{72\\}\\)\\(.+\\)$" nil t) + (replace-match "\\1\n \\2") + (goto-char (line-beginning-position))))) + + (let ((ical-name gnus-icalendar-reply-bufname)) + (with-current-buffer (get-buffer-create ical-name) + (delete-region (point-min) (point-max)) + (insert reply) + (fold-icalendar-buffer) + (when (and charset (string= (downcase charset) "utf-8")) + (decode-coding-region (point-min) (point-max) 'utf-8))) + + (save-excursion ;; Compose the reply message. + (let* ((message-signature nil) + (organizer (gnus-icalendar-event:organizer event)) + (organizer (when (and organizer + (not (string-empty-p organizer))) + organizer)) + (organizer + (or organizer + (plist-get (car (plist-get msg :reply-to)) :email) + (plist-get (car (plist-get msg :from)) :email) + (mu4e-warn "Cannot find organizer"))) + (message-cite-function #'mu4e-message-cite-nothing)) + (mu4e-compose-reply-to organizer) + (message-goto-body) + (mml-insert-multipart "alternative") + (mml-insert-empty-tag 'part 'type "text/plain") + (mml-attach-buffer ical-name + "text/calendar; method=REPLY; charset=UTF-8") + (when mu4e-icalendar-trash-after-reply + ;; Override `mu4e-sent-handler' set by `mu4e-compose-mode' to + ;; also trash the message (thus must be appended to hooks). + (add-hook 'message-sent-hook + (mu4e--icalendar-trash-message-hook msg) 90 t)) + + (when gnus-icalendar-org-enabled-p + (if (gnus-icalendar-find-org-event-file event) + (gnus-icalendar--update-org-event event status) + (gnus-icalendar:org-event-save event status))) + (when mu4e-icalendar-diary-file + (mu4e--icalendar-insert-diary event status + mu4e-icalendar-diary-file))))))))) + +(declare-function mu4e-view-headers-next "mu4e-view") +(defun mu4e--icalendar-trash-message (original-msg) + "Trash the message ORIGINAL-MSG and move to the next one." + (lambda (docid path) + "See `mu4e-sent-handler' for DOCID and PATH." + (mu4e-sent-handler docid path) + (let* ((docid (mu4e-message-field original-msg :docid)) + (markdescr (assq 'trash mu4e-marks)) + (action (plist-get (cdr markdescr) :action)) + (target (mu4e-get-trash-folder original-msg))) + (with-current-buffer (mu4e-get-headers-buffer) + (run-hook-with-args 'mu4e-mark-execute-pre-hook 'trash original-msg) + (funcall action docid original-msg target)) + (when (and (mu4e~headers-view-this-message-p docid) + (buffer-live-p (mu4e-get-view-buffer))) + (mu4e-display-buffer (mu4e-get-view-buffer)) + (or (mu4e-view-headers-next) + (kill-buffer-and-window)))))) + +(defun mu4e--icalendar-trash-message-hook (original-msg) + "Trash the iCalendar message ORIGINAL-MSG." + (lambda () + (setq mu4e-sent-func + (mu4e--icalendar-trash-message original-msg)))) + +(defun mu4e--icalendar-insert-diary (event reply-status filename) + "Insert a diary entry for the EVENT in file named FILENAME. +REPLY-STATUS is the status of the reply. The possible values are +given in the doc of `gnus-icalendar-event-reply-from-buffer'." + ;; FIXME: handle recurring events + (let* ((beg (gnus-icalendar-event:start-time event)) + (beg-date (format-time-string "%d/%m/%Y" beg)) + (beg-time (format-time-string "%H:%M" beg)) + (end (gnus-icalendar-event:end-time event)) + (end-date (format-time-string "%d/%m/%Y" end)) + (end-time (format-time-string "%H:%M" end)) + (summary (gnus-icalendar-event:summary event)) + (location (gnus-icalendar-event:location event)) + (status (capitalize (symbol-name reply-status))) + (txt (if location + (format "%s (%s)\n %s " summary status location) + (format "%s (%s)" summary status)))) + (with-temp-buffer + (if (string= beg-date end-date) + (insert beg-date " " beg-time "-" end-time " " txt "\n") + (insert beg-date " " beg-time " Start of: " txt "\n") + (insert beg-date " " end-time " End of: " txt "\n")) + (write-region (point-min) (point-max) filename t)))) + +;;; +(provide 'mu4e-icalendar) +;;; mu4e-icalendar.el ends here diff --git a/mu4e/mu4e-lists.el b/mu4e/mu4e-lists.el new file mode 100644 index 0000000..f19239f --- /dev/null +++ b/mu4e/mu4e-lists.el @@ -0,0 +1,170 @@ +;;; mu4e-lists.el --- Get names for mailing lists -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; In this file, we create a table of list-id -> shortname for mailing lists. +;; The shortname (friendly) should a at most 8 characters, camel-case + +;;; Code: +(require 'mu4e-message) +(require 'mu4e-helpers) + + + ;;; Helpers +(defmacro mu4e-message-id-url(base-url) + "Construct lambda to get an archive URL for message. +This is based on some BASE-URL to which the message-id is concatenated; +e.g. public-inbox-based archives." + `(lambda (msg) (concat ,base-url "/" (plist-get msg :message-id)))) + +(defmacro mu4e-x-seq-url (base-url) + "Construct x-seq archive URL for MSG or nil if not found." + `(lambda (msg) + (when-let ((xseq (mu4e-fetch-field msg "X-Seq"))) + (concat ,base-url "/" xseq)))) + +;;; Configuration +(defvar mu4e-mailing-lists + `( (:list-id "bbdb-info.lists.sourceforge.net" :name "BBDB") + (:list-id "boost-announce.lists.boost.org" :name "Boost") + (:list-id "boost-interest.lists.boost.org" :name "Boost") + (:list-id "curl-library.cool.haxx.se" :name "Curl") + (:list-id "dbus.lists.freedesktop.org" :name "DBus") + (:list-id "desktop-devel-list.gnome.org" :name "Gnome") + (:list-id "discuss-webrtc.googlegroups.com" :name "WebRTC") + (:list-id "emacs-devel.gnu.org" :name "EmacsDev" + :archive ,(mu4e-message-id-url "https://yhetil.org/emacs-devel")) + (:list-id "emacs-orgmode.gnu.org" :name "Orgmode" + :archive ,(mu4e-message-id-url "https://list.orgmode.org")) + (:list-id "emms-help.gnu.org" :name "Emms") + (:list-id "gcc-help.gcc.gnu.org" :name "Gcc") + (:list-id "gmime-devel-list.gnome.org" :name "GMime") + (:list-id "gnome-shell-list.gnome.org" :name "Gnome") + (:list-id "gnu-emacs-sources.gnu.org" :name "Emacs") + (:list-id "gnupg-users.gnupg.org" :name "Gnupg") + (:list-id "gstreamer-devel.lists.freedesktop.org" :name "GstDev") + (:list-id "gtk-devel-list.gnome.org" :name "GtkDev") + (:list-id "guile-devel.gnu.org" :name "Guile" + :archive ,(mu4e-message-id-url "https://yhetil.org/guile-devel")) + (:list-id "guile-user.gnu.org" :name "Guile" + :archive ,(mu4e-message-id-url "https://yhetil.org/guile-user")) + (:list-id "help-gnu-emacs.gnu.org" :name "EmacsUsr" + :archive ,(mu4e-message-id-url "https://yhetil.org/emacs-user")) + (:list-id "mu-discuss.googlegroups.com" :name "Mu") + (:list-id "nautilus-list.gnome.org" :name "Nautilus") + (:list-id "notmuch.notmuchmail.org" :name "Notmuch" + :archive ,(mu4e-message-id-url "https://yhetil.org/notmuch")) + (:list-id "sqlite-announce.sqlite.org" :name "SQlite") + (:list-id "sqlite-dev.sqlite.org" :name "SQLite") + (:list-id "xapian-discuss.lists.xapian.org" :name "Xapian") + (:list-id "xdg.lists.freedesktop.org" :name "XDG") + (:list-id "wl-en.lists.airs.net" :name "WdrLust") + (:list-id "wl-en.ml.gentei.org" :name "WdrLust") + (:list-id "xapian-devel.lists.xapian.org" :name "Xapian") + (:list-id "zsh-users.zsh.org" :name "Zsh" + :archive ,(mu4e-x-seq-url "https://www.zsh.org/users"))) + "List of plists with keys: +- `:list-id' - the mailing list id +- `:name' - the display name +- `:archive' - (optional) a function taking a MSG and + returning an URL to to the online-location of + the message. +After changes, use `mu4e-mailing-list-info-refresh' to update the +corresponding data-structures.") + +(defgroup mu4e-lists nil "Configuration for mailing lists." + :group 'mu4e) + +(defcustom mu4e-user-mailing-lists nil + "A list with plists like `mu4e-mailing-lists'. +These are used in addition to the built-in list +`mu4e-mailing-lists'. + +The older format, a list of cons cells, + (LIST-ID . NAME) +is still supported for backward compatibility. + +After changing, use `mu4e-mailing-list-info-refresh' to make mu4e +use the new values." + :group 'mu4e-headers + :type '(repeat (plist))) + +(defcustom mu4e-mailing-list-patterns '("\\([^.]*\\)\\.") + "A list of regexps to capture a shortname out of a list-id. +For the first regex that matches, its first match-group will be +used as the shortname." + :group 'mu4e-headers + :type '(repeat (regexp))) + +(defvar mu4e--lists-hash nil + "Hash-table of list-id => plist. +Based on `mu4e-mailing-lists' and `mu4e-user-mailing-lists'.") + +(defun mu4e-mailing-list-info-refresh () + "Refresh the mailing list info. +Based on the current value of `mu4e-mailing-lists' and +`mu4e-user-mailing-lists'." + (interactive) + (setq mu4e--lists-hash (make-hash-table :test 'equal)) + (seq-do (lambda (item) + (if (mu4e-plistp item) + ;; the new format + (puthash (plist-get item :list-id) item mu4e--lists-hash) + ;; backward compatibility + (puthash (car item) (cdr item) mu4e--lists-hash))) + (append mu4e-mailing-lists + mu4e-user-mailing-lists)) + mu4e--lists-hash) + +(defun mu4e-mailing-list-info (list-id) + "Get mailing list info for LIST-ID. +Return nil if not found." + (unless mu4e--lists-hash (mu4e-mailing-list-info-refresh)) + (gethash list-id mu4e--lists-hash)) + + +(defun mu4e-get-mailing-list-shortname (list-id) + "Get the shortname for a mailing-list with list-id LIST-ID. +Either we know about this mailing list, or otherwise +we guess one." + (or ;; 1. perhaps we have it in one of our lists? + (plist-get (mu4e-mailing-list-info list-id) :name) + ;; 2. see if it matches some pattern + (if (seq-find (lambda (p) (string-match p list-id)) + mu4e-mailing-list-patterns) + (match-string 1 list-id) + ;; 3. otherwise, just return the whole thing + list-id))) + +(defun mu4e-mailing-list-archive-url (&optional msg) + "Get the mailing-list archive URL for MSG. +If MSG is nil, use the message at point." + (when-let* ((msg (or msg (mu4e-message-at-point))) + (list-id (plist-get msg :list)) + (list-info (and list-id (mu4e-mailing-list-info list-id))) + (func (plist-get list-info :archive))) + (when func + (funcall func msg)))) + +(provide 'mu4e-lists) +;;; mu4e-lists.el ends here diff --git a/mu4e/mu4e-main.el b/mu4e/mu4e-main.el new file mode 100644 index 0000000..ffe22a5 --- /dev/null +++ b/mu4e/mu4e-main.el @@ -0,0 +1,435 @@ +;;; mu4e-main.el --- The Main interface for mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: + +(require 'smtpmail) +(require 'mu4e-helpers) +(require 'mu4e-context) +(require 'mu4e-compose) +(require 'mu4e-bookmarks) +(require 'mu4e-folders) +(require 'mu4e-update) +(require 'mu4e-contacts) +(require 'mu4e-search) +(require 'mu4e-vars) ;; mu-wide variables +(require 'mu4e-window) +(require 'mu4e-query-items) + +(declare-function mu4e-compose-new "mu4e-compose") +(declare-function mu4e-quit "mu4e") + +(require 'cl-lib) + + +;; Configuration + +(defcustom mu4e-main-hide-personal-addresses nil + "Whether to hide the personal address in the main view. + + This can be useful to avoid the noise when there are many, and +also hides the warning if your `user-mail-address' is not part of +the personal addresses." + :type 'boolean + :group 'mu4e-main) + +(defcustom mu4e-main-hide-fully-read nil + "Whether to hide bookmarks or maildirs without unread messages." + :type 'boolean + :group 'mu4e-main) + +(defcustom mu4e-main-rendered-hook nil + "Hook run after the main-view has been rendered." + :type 'hook + :group 'mu4e-main) + + +;;; Mode +(define-derived-mode mu4e-org-mode org-mode "mu4e:org" + "Major mode for mu4e documents.") + +(defun mu4e-info (path) + "Show a buffer with the information (an org-file) at PATH." + (unless (file-exists-p path) + (mu4e-error "Cannot find %s" path)) + (let ((curbuf (current-buffer))) + (find-file path) + (mu4e-org-mode) + (setq buffer-read-only t) + (define-key mu4e-org-mode-map (kbd "q") + `(lambda () + (interactive) + (bury-buffer) + (switch-to-buffer ,curbuf))))) + +(defun mu4e-about () + "Show the mu4e \"About\" page." + (interactive) + (mu4e-info (mu4e-join-paths mu4e-doc-dir "mu4e-about.org"))) + +(defun mu4e-news () + "Show page with news for the current version of mu4e." + (interactive) + (mu4e-info (mu4e-join-paths mu4e-doc-dir "NEWS.org"))) + +(defun mu4e-baseline-time () + "Show the baseline time." + (interactive) + (mu4e-message "Baseline time: %s" (mu4e--baseline-time-string))) + +(defun mu4e--baseline-time-string () + "Calculate the baseline time string." + (let* ((baseline-t mu4e--query-items-baseline-tstamp) + (updated-t (plist-get mu4e-index-update-status :tstamp)) + (delta-t (and baseline-t updated-t + (float-time (time-subtract updated-t baseline-t))))) + (if (and delta-t (> delta-t 0)) + (format-seconds "%Y %D %H %M %z%S since latest" delta-t) + (if baseline-t + (current-time-string baseline-t) + "Never")))) + +(defvar mu4e-main-mode-map + (let ((map (make-sparse-keymap))) + + (define-key map "q" #'mu4e-quit) + (define-key map "C" #'mu4e-compose-new) + + (define-key map "m" #'mu4e--main-toggle-mail-sending-mode) + (define-key map "f" #'smtpmail-send-queued-mail) + ;; + (define-key map (kbd "C-S-u") #'mu4e-update-mail-and-index) + ;; for terminal users + (define-key map (kbd "C-c C-u") #'mu4e-update-mail-and-index) + (define-key map "U" #'mu4e-update-mail-and-index) + (define-key map "S" #'mu4e-kill-update-mail) + (define-key map ";" #'mu4e-context-switch) + (define-key map "$" #'mu4e-show-log) + (define-key map "A" #'mu4e-about) + (define-key map "N" #'mu4e-news) + (define-key map "H" #'mu4e-display-manual) + map) + "Keymap for the *mu4e-main* buffer.") + +(easy-menu-define mu4e-main-mode-menu + mu4e-main-mode-map "Menu for mu4e's main view." + (append + '("Mu4e" ;;:visible mu4e-headers-mode + "--" + ["Update mail and index" mu4e-update-mail-and-index] + ["Flush queued mail" smtpmail-send-queued-mail] + "--" + ["Show debug log" mu4e-show-log] + ) + mu4e--compose-menu-items + mu4e--search-menu-items + '( + "--" + ["Quit" mu4e-quit :help "Quit mu4e"]))) + +(declare-function mu4e--server-bookmarks-queries "mu4e") + +(define-derived-mode mu4e-main-mode special-mode "mu4e:main" + "Major mode for the mu4e main screen. + +This mode is a bit special when it comes to keybinding, since it +shows those keybindings. + +For the rebinding the mu4e functions (such as +`mu4e-search-bookmark' and `mu4e-search-maildir') to different +keys, note that mu4e determines the bindings when drawing the +screen, which is *after* we enable the mode. Thus, the +keybindings must be known when this happens. + +Binding the existing bindings (such as \='s') to different +functions, is *not* really supported, and we still display the +default binding for the original function; which should still do +the reasonable thing in most cases. + +Still, such a rebinding *only* affects the key, and not e.g. the +mouse-bindings." + (setq truncate-lines t + overwrite-mode 'overwrite-mode-binary) + (mu4e-context-minor-mode) + (mu4e-search-minor-mode) + (mu4e-update-minor-mode) + (setq-local revert-buffer-function + (lambda (_ignore-auto _noconfirm) + ;; reset the baseline and get updated results. + (mu4e--query-items-refresh 'reset-baseline)))) + + +(defun mu4e--main-action (title cmd &optional bindstr alt) + "Produce main view action string with TITLE. + +When activated, invoke interactive function CMD. + +In the result, used the TITLE string, with the first occurrence +of [@] replaced by a textual replacement of a binding to CMD as +per `mu4e-key-description', or, if specified, BINDSTR. + +If a string ALT is specified, and BINDSTR is longer than a single +character, use ALT as a substitute. ALT should be a string of +length 1. + +If the first letter after the [@] is equal to the last letter of the +binding representation, remove that first letter." + (let* ((bindstr (or bindstr (mu4e-key-description cmd) alt + (mu4e-error "No binding for %s" cmd))) + (bindstr + (if (and alt (> (length bindstr) 1)) alt bindstr)) + (title ;; remove first letter afrer [] if it equal last of binding + (mu4e-string-replace + (concat "[@]" (substring bindstr -1)) "[@]" title)) + (title ;; insert binding in [@] + (mu4e-string-replace + "[@]" (format "[%s]" (propertize bindstr 'face 'mu4e-highlight-face)) + title)) + (map (make-sparse-keymap))) + (define-key map [mouse-2] cmd) + (define-key map (kbd "RET") cmd) + (propertize title 'keymap map))) + +(defun mu4e--main-items (item-type max-length) + "Produce the string with menu-items for ITEM-TYPE. +ITEM-TYPE is a symbol, either `bookmarks' or `maildirs'. + +MAX-LENGTH is the maximum length of the item titles; this is used +for aligning them." + (mapconcat + (lambda (item) + (cl-destructuring-bind + (&key hide name key favorite query &allow-other-keys) item + ;; hide items explicitly hidden, without key or wrong category. + (if hide + "" + (let ((item-info + ;; note, we have a function for the binding, + ;; and perhaps a different one for the lambda. + (cond + ((eq item-type 'maildirs) + (list #'mu4e-search-maildir #'mu4e-search + query)) + ((eq item-type 'bookmarks) + (list #'mu4e-search-bookmark #'mu4e-search-bookmark + (mu4e-get-bookmark-query key))) + (t + (mu4e-error "Invalid item-type %s" item-type))))) + (concat + (mu4e--main-action + ;; main title + (format "\t* [@] %s " + (propertize + name + 'face (if favorite 'mu4e-header-key-face nil) + 'help-echo query)) + ;; function to call when activated + (lambda () (interactive) + (funcall (nth 1 item-info) + (nth 2 item-info))) + ;; custom key binding string + (concat (mu4e-key-description (nth 0 item-info)) (string key))) + ;; counts + (format "%s%s\n" + (make-string (- max-length (string-width name)) ?\s) + (mu4e--query-item-display-counts item))))))) + ;; only items which have a single-character :key + (mu4e-filter-single-key (mu4e-query-items item-type)) "")) + +(defun mu4e--key-val (key val &optional unit) + "Show a KEY / VAL pair, with optional UNIT." + (concat + "\t* " + (propertize (format "%-20s" key) 'face 'mu4e-header-title-face) + ": " + (propertize val 'face 'mu4e-header-key-face) + (if unit + (propertize (concat " " unit) 'face 'mu4e-header-title-face) + "") + "\n")) + +(defun mu4e--main-baseline-time-string () + "Calculate the baseline time string for use in the main-" + (let* ((baseline-t mu4e--query-items-baseline-tstamp) + (updated-t (plist-get mu4e-index-update-status :tstamp)) + (delta-t (and baseline-t updated-t + (float-time (time-subtract updated-t baseline-t))))) + (if (and delta-t (> delta-t 0)) + (format-seconds "%Y %D %H %M %z%S ago" delta-t) + (if baseline-t + (current-time-string baseline-t) + "Never")))) + +(defun mu4e--main-redraw () + "Redraw the main buffer if there is one. +Otherwise, do nothing." + (when-let* ((buffer (get-buffer mu4e-main-buffer-name)) + (buffer (and (buffer-live-p buffer) buffer))) + (with-current-buffer buffer + (let* ((inhibit-read-only t) + (pos (point)) + (addrs (mu4e-personal-addresses)) + (max-length (seq-reduce (lambda (a b) + (max a (length (plist-get b :name)))) + (mu4e-query-items) 0))) + (mu4e-main-mode) + (erase-buffer) + (insert + "* " + (propertize "mu4e" 'face 'mu4e-header-key-face) + (propertize " - mu for emacs version " 'face 'mu4e-title-face) + (propertize mu4e-mu-version 'face 'mu4e-header-key-face) + "\n\n" + (propertize " Basics\n\n" 'face 'mu4e-title-face) + (mu4e--main-action + "\t* [@]jump to some maildir\n" #'mu4e-search-maildir nil "j") + (mu4e--main-action + "\t* enter a [@]search query\n" #'mu4e-search nil "s") + (mu4e--main-action + "\t* [@]Compose a new message\n" #'mu4e-compose-new nil "C") + "\n" + (propertize " Bookmarks\n\n" 'face 'mu4e-title-face) + (mu4e--main-items 'bookmarks max-length) + "\n" + (propertize " Maildirs\n\n" 'face 'mu4e-title-face) + (mu4e--main-items 'maildirs max-length) + "\n" + (propertize " Misc\n\n" 'face 'mu4e-title-face) + (mu4e--main-action "\t* [@]Choose query\n" + #'mu4e-search-query nil "c") + (mu4e--main-action "\t* [@]Switch context\n" + #'mu4e-context-switch nil ";") + (mu4e--main-action "\t* [@]Update email & database\n" + #'mu4e-update-mail-and-index nil "U") + ;; show the queue functions if `smtpmail-queue-dir' is defined + (if (file-directory-p smtpmail-queue-dir) + (mu4e--main-view-queue) + "") + "\n" + (mu4e--main-action "\t* [@]News\n" #'mu4e-news nil "N") + (mu4e--main-action "\t* [@]About mu4e\n" #'mu4e-about nil "A") + (mu4e--main-action "\t* [@]Help\n" #'mu4e-display-manual nil "H") + (mu4e--main-action "\t* [@]quit\n" #'mu4e-quit nil "q") + "\n" + (propertize " Info\n\n" 'face 'mu4e-title-face) + (mu4e--key-val "last updated" + (current-time-string + (plist-get mu4e-index-update-status :tstamp))) + (mu4e--key-val "database-path" (mu4e-database-path)) + (mu4e--key-val "maildir" (mu4e-root-maildir)) + (mu4e--key-val "in store" + (format "%d" (plist-get mu4e--server-props :doccount)) + "messages") + (if mu4e-main-hide-personal-addresses "" + (mu4e--key-val "personal addresses" + (if addrs (mapconcat #'identity addrs ", " ) "none")))) + + (if mu4e-main-hide-personal-addresses "" + (unless (mu4e-personal-address-p user-mail-address) + (mu4e-message (concat + "Tip: `user-mail-address' ('%s') is not part " + "of mu's addresses; add it with 'mu init + --my-address='") user-mail-address))) + (goto-char pos))))) + +(defun mu4e--main-view-queue () + "Display queue-related actions in the main view." + (concat + (mu4e--main-action "\t* toggle [@]mail sending mode " + #'mu4e--main-toggle-mail-sending-mode) + "(currently " + (propertize (if smtpmail-queue-mail "queued" "direct") + 'face 'mu4e-header-key-face) + ")\n" + (let ((queue-size (mu4e--main-queue-size))) + (if (zerop queue-size) + "" + (mu4e--main-action + (format "\t* [@]flush %s queued %s\n" + (propertize (int-to-string queue-size) + 'face 'mu4e-header-key-face) + (if (> queue-size 1) "mails" "mail")) + 'smtpmail-send-queued-mail))))) + +(defun mu4e--main-queue-size () + "Return, as an int, the number of emails in the queue." + (condition-case nil + (with-temp-buffer + (insert-file-contents (expand-file-name smtpmail-queue-index-file + smtpmail-queue-dir)) + (count-lines (point-min) (point-max))) + (error 0))) + +(declare-function mu4e--start "mu4e") + +(defun mu4e--main-view () + "(Re)create the mu4e main-view, and switch to it. + +If `mu4e-split-view' equals \\='single-window, show a mu4e menu +instead." + (if (eq mu4e-split-view 'single-window) + (mu4e--main-menu) + (let ((buf (get-buffer-create mu4e-main-buffer-name)) + (inhibit-read-only t)) + (with-current-buffer buf + (mu4e--main-redraw)) + (mu4e-display-buffer buf t) + (run-hooks 'mu4e-main-rendered-hook))) + (goto-char (point-min))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Interactive functions +;; Toggle mail sending mode without switching +(defun mu4e--main-toggle-mail-sending-mode () + "Toggle sending mail mode, either queued or direct." + (interactive) + (unless (file-directory-p smtpmail-queue-dir) + (mu4e-error "`smtpmail-queue-dir' does not exist")) + (setq smtpmail-queue-mail (not smtpmail-queue-mail)) + (message (concat "Outgoing mail will now be " + (if smtpmail-queue-mail "queued" "sent directly"))) + (unless (or (eq mu4e-split-view 'single-window) + (not (buffer-live-p (get-buffer mu4e-main-buffer-name)))) + (mu4e--main-redraw))) + +(defun mu4e--main-menu () + "The mu4e main menu in the mini-buffer." + (let ((func (mu4e-read-option + "Do: " + '(("jump" . mu4e~headers-jump-to-maildir) + ("search" . mu4e-search) + ("Compose" . mu4e-compose-new) + ("bookmarks" . mu4e-search-bookmark) + (";Switch context" . mu4e-context-switch) + ("Update" . mu4e-update-mail-and-index) + ("News" . mu4e-news) + ("About" . mu4e-about) + ("Help " . mu4e-display-manual))))) + (call-interactively func) + (when (eq func 'mu4e-context-switch) + (sit-for 1) + (mu4e--main-menu)))) + +(provide 'mu4e-main) +;;; mu4e-main.el ends here diff --git a/mu4e/mu4e-mark.el b/mu4e/mu4e-mark.el new file mode 100644 index 0000000..c3947a7 --- /dev/null +++ b/mu4e/mu4e-mark.el @@ -0,0 +1,471 @@ +;;; mu4e-mark.el --- Marking messages -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; In this file are function related to marking messages; they assume we are +;; currently in the headers buffer. + +;;; Code: + +(require 'mu4e-server) +(require 'mu4e-message) +(require 'mu4e-folders) + +;; keep byte-compiler happy +(declare-function mu4e~headers-mark "mu4e-headers") +(declare-function mu4e~headers-goto-docid "mu4e-headers") +(declare-function mu4e-headers-next "mu4e-headers") + +;;; Variables & constants + +(defcustom mu4e-headers-leave-behavior 'ask + "What to do when user leaves the current headers view. + +\"Leaving\" here means quitting the headers views, refreshing it +or even quitting mu4e or Emacs. + +Value is one of the following symbols: +- `ask' ask user whether to ignore the marks +- `apply' automatically apply the marks before doing anything else +- `ignore' automatically ignore the marks without asking" + :type '(choice (const :tag "ask user whether to ignore marks" ask) + (const :tag "apply marks without asking" apply) + (const :tag "ignore marks without asking" ignore)) + :group 'mu4e-headers) + +(defcustom mu4e-mark-execute-pre-hook nil + "Hook run just *before* a mark is applied to a message. +The hook function is called with two arguments, the mark being +executed and the message itself." + :type 'hook + :group 'mu4e-headers) + +(defvar mu4e-headers-show-target t + "Whether to show targets (such as \"-> delete\", \"-> /archive\") +when marking message. Normally, this is useful information for +the user, however, when you often mark large numbers (thousands) +of message, showing the target makes this quite a bit +slower (showing the target uses Emacs overlays, which can be slow +when overused).") + +;;; Insert stuff + +(defvar mu4e--mark-map nil + "Contains a mapping of docid->markinfo. +When a message is marked, the information is added here. markinfo +is a cons cell consisting of the following: (mark . target) where +MARK is the type of mark (move, trash, delete) TARGET (optional) +is the target directory (for \"move\")") + +;; the mark-map is specific for the current header buffer +;; currently, there can't be more than one, but we never know what will +;; happen in the future + +;; the fringe is the space on the left of headers, where we put marks below some +;; handy definitions; only `mu4e-mark-fringe-len' should be change (if ever), +;; the others follow from that. +(defconst mu4e--mark-fringe-len 2 + "Width of the fringe for marks on the left.") +(defconst mu4e--mark-fringe (make-string mu4e--mark-fringe-len ?\s) + "The space on the left of message headers to put marks.") +(defconst mu4e--mark-fringe-format (format "%%-%ds" mu4e--mark-fringe-len) + "Format string to set a mark and leave remaining space.") + +(defun mu4e--mark-initialize () + "Initialize the marks-subsystem." + (set (make-local-variable 'mu4e--mark-map) (make-hash-table)) + ;; ask user when kill buffer / emacs with live marks. + ;; (subject to mu4e-headers-leave-behavior) + (add-hook 'kill-buffer-query-functions + #'mu4e-mark-handle-when-leaving nil t) + (add-hook 'kill-emacs-query-functions + #'mu4e-mark-handle-when-leaving nil t)) + +(defun mu4e--mark-clear () + "Clear the marks-subsystem." + (clrhash mu4e--mark-map)) + +(defun mu4e--mark-find-headers-buffer () + "Find the headers buffer, if any." + (seq-find (lambda (_) + (mu4e-current-buffer-type-p 'headers)) + (buffer-list))) + +(defmacro mu4e--mark-in-context (&rest body) + "Evaluate BODY in the context of the headers buffer. +The current buffer must be either a headers or view buffer." + `(cond + ((mu4e-current-buffer-type-p 'headers) ,@body) + ((mu4e-current-buffer-type-p 'view) + (when (buffer-live-p (mu4e-get-headers-buffer)) + (let* ((msg (mu4e-message-at-point)) + (docid (mu4e-message-field msg :docid))) + (with-current-buffer (mu4e-get-headers-buffer) + (when (mu4e~headers-goto-docid docid) + ,@body + ))))))) + +(defconst mu4e-marks + '((refile + :char ("r" . "▶") + :prompt "refile" + :dyn-target (lambda (target msg) (mu4e-get-refile-folder msg)) + :action (lambda (docid msg target) + (mu4e--server-move docid (mu4e--mark-check-target target) "-N"))) + (delete + :char ("D" . "x") + :prompt "Delete" + :show-target (lambda (target) "delete") + :action (lambda (docid msg target) (mu4e--server-remove docid))) + (flag + :char ("+" . "✚") + :prompt "+flag" + :show-target (lambda (target) "flag") + :action (lambda (docid msg target) + (mu4e--server-move docid nil "+F-u-N"))) + (move + :char ("m" . "▷") + :prompt "move" + :ask-target mu4e--mark-get-move-target + :action (lambda (docid msg target) + (mu4e--server-move docid (mu4e--mark-check-target target) "-N"))) + (read + :char ("!" . "◼") + :prompt "!read" + :show-target (lambda (target) "read") + :action (lambda (docid msg target) (mu4e--server-move docid nil "+S-u-N"))) + (trash + :char ("d" . "▼") + :prompt "dtrash" + :dyn-target (lambda (target msg) (mu4e-get-trash-folder msg)) + :action (lambda (docid msg target) + (mu4e--server-move docid + (mu4e--mark-check-target target) "+T-N"))) + (unflag + :char ("-" . "➖") + :prompt "-unflag" + :show-target (lambda (target) "unflag") + :action (lambda (docid msg target) (mu4e--server-move docid nil "-F-N"))) + (untrash + :char ("=" . "▲") + :prompt "=untrash" + :show-target (lambda (target) "untrash") + :action (lambda (docid msg target) (mu4e--server-move docid nil "-T"))) + (unread + :char ("?" . "◻") + :prompt "?unread" + :show-target (lambda (target) "unread") + :action (lambda (docid msg target) (mu4e--server-move docid nil "-S+u-N"))) + (unmark + :char " " + :prompt "unmark" + :action (mu4e-error "No action for unmarking")) + (action + :char ( "a" . "◯") + :prompt "action" + :ask-target (lambda () (mu4e-read-option "Action: " mu4e-headers-actions)) + :action (lambda (docid msg actionfunc) + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-headers-action actionfunc))))) + (something + :char ("*" . "✱") + :prompt "*something" + :action (mu4e-error "No action for deferred mark"))) + + "The list of all the possible marks. +This is an alist mapping mark symbols to their properties. The +properties are: + :char (string) or (basic . fancy) The character to display in + the headers view. Either a single-character string, or a + dotted-pair cons cell where the second item will be used if + `mu4e-use-fancy-chars' is t, otherwise we'll use + the first one. It can also be a plain string for backwards + compatibility since we didn't always support + `mu4e-use-fancy-chars' here. + :prompt (string) The prompt to use when asking for marks (used for + example when marking a whole thread) + :ask-target (function returning a string) Get the target. This + function run once per bulk-operation, and thus is suitable + for user-interaction. If nil, the target is nil. + :dyn-target (function from (TARGET MSG) to string). Compute + the dynamic target. This is run once per message, which is + passed as MSG. The default is to just return the target. + :show-target (function from TARGET to string) How to display + the target. + :action (function taking (DOCID MSG TARGET)). The action to + apply on the message.") + +(defun mu4e-mark-at-point (mark target) + "Mark (or unmark) message at point. +MARK specifies the mark-type. For `move'-marks and `trash'-marks +the TARGET argument is non-nil and specifies to which maildir the +message is to be moved/trashed. The function works in both +headers buffers and message buffers. + +The following marks are available, and the corresponding props: + + MARK TARGET description + ---------------------------------------------------------- + `refile' y mark this message for archiving + `something' n mark this message for *something* (decided later) + `delete' n remove the message + `flag' n mark this message for flagging + `move' y move the message to some folder + `read' n mark the message as read + `trash' y trash the message to some folder + `unflag' n mark this message for unflagging + `untrash' n remove the `trashed' flag from a message + `unmark' n unmark this message + `unread' n mark the message as unread + `action' y mark the message for some action." + (interactive) + (let* ((msg (mu4e-message-at-point)) + (docid (mu4e-message-field msg :docid)) + ;; get a cell with the mark char and the "move" already has a target + ;; (the target folder) the other ones get a pseudo "target", as info + ;; for the user. + (markdesc (cdr (or (assq mark mu4e-marks) + (mu4e-error "Invalid mark %S" mark)))) + (get-markkar + (lambda (char) + (if (listp char) + (if mu4e-use-fancy-chars (cdr char) (car char)) + char))) + (markkar (funcall get-markkar (plist-get markdesc :char))) + (target (mu4e--mark-get-dyn-target mark target)) + (show-fct (plist-get markdesc :show-target)) + (shown-target (if show-fct + (funcall show-fct target) + (if target (format "%S" target))))) + (unless docid (mu4e-warn "No message on this line")) + (unless (eq major-mode 'mu4e-headers-mode) + (mu4e-error "Not in headers-mode")) + (save-excursion + (when (mu4e~headers-mark docid markkar) + ;; update the hash -- remove everything current, and if add the new + ;; stuff, unless we're unmarking + (remhash docid mu4e--mark-map) + ;; remove possible mark overlays + (remove-overlays (line-beginning-position) (line-end-position) + 'mu4e-mark t) + ;; now, let's set a mark (unless we were unmarking) + (unless (eql mark 'unmark) + (puthash docid (cons mark target) mu4e--mark-map) + ;; when we have a target (ie., when moving), show the target folder in + ;; an overlay + (when (and shown-target mu4e-headers-show-target) + (let* ((targetstr (propertize (concat "-> " shown-target " ") + 'face 'mu4e-system-face)) + ;; mu4e~headers-goto-docid docid t \will take us just after + ;; the docid cookie and then we skip the mu4e--mark-fringe + (start (+ (length mu4e--mark-fringe) + (mu4e~headers-goto-docid docid t))) + (overlay (make-overlay start (+ start (length targetstr))))) + (overlay-put overlay 'display targetstr) + (overlay-put overlay 'mu4e-mark t) + (overlay-put overlay 'evaporate t) + docid))))))) + +(defun mu4e--mark-get-move-target () + "Ask for a move target, and propose to create it if it does not exist." + (let* ((target (mu4e-ask-maildir "Move message to: ")) + (target (if (string= (substring target 0 1) "/") + target + (concat "/" target))) + (fulltarget (mu4e-join-paths (mu4e-root-maildir) target))) + (when (mu4e-create-maildir-maybe fulltarget) + target))) + +(defun mu4e--mark-ask-target (mark) + "Ask the target for MARK, if the user should be asked the target." + (let ((getter (plist-get (cdr (assq mark mu4e-marks)) :ask-target))) + (and getter (funcall getter)))) + +(defun mu4e--mark-get-dyn-target (mark target) + "Get the dynamic TARGET for MARK. +The result may depend on the message at point." + (let ((getter (plist-get (cdr (assq mark mu4e-marks)) :dyn-target))) + (if getter + (funcall getter target (mu4e-message-at-point)) + target))) + +(defun mu4e-mark-set (mark &optional target) + "Mark the header at point with MARK or all in the region. +Optionally, provide TARGET (for moves)." + (unless target + (setq target (mu4e--mark-ask-target mark))) + (if (not (use-region-p)) + ;; single message + (mu4e-mark-at-point mark target) + ;; mark all messages in the region. + (save-excursion + (let ((cant-go-further) (eor (region-end))) + (goto-char (region-beginning)) + (while (and (< (point) eor) (not cant-go-further)) + (mu4e-mark-at-point mark target) + (setq cant-go-further (not (mu4e-headers-next)))))))) + +(defun mu4e-mark-restore (docid) + "Restore the visual mark for the message with DOCID." + (let ((markcell (gethash docid mu4e--mark-map))) + (when markcell + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-mark-at-point (car markcell) (cdr markcell))))))) + +(defun mu4e--mark-get-markpair (prompt &optional allow-something) + "Ask user with PROMPT for a mark and return (MARK . TARGET). +If ALLOW-SOMETHING is non-nil, allow the `something' pseudo mark +as well." + (let* ((marks (mapcar (lambda (markdescr) + (cons (plist-get (cdr markdescr) :prompt) + (car markdescr))) + mu4e-marks)) + (marks + (if allow-something + marks (seq-remove (lambda (m) (eq 'something (cdr m))) marks))) + (mark (mu4e-read-option prompt marks)) + (target (mu4e--mark-ask-target mark))) + (cons mark target))) + +(defun mu4e-mark-resolve-deferred-marks () + "Check if there are any deferred ('something') mark-instances. +If there are such marks, replace them with a _real_ mark (ask the +user which one)." + (interactive) + (mu4e--mark-in-context + (let ((markpair)) + (maphash + (lambda (docid val) + (let ((mark (car val))) + (when (eql mark 'something) + (unless markpair + (setq markpair + (mu4e--mark-get-markpair "Set deferred mark(s) to: " nil))) + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-mark-set (car markpair) (cdr markpair))))))) + mu4e--mark-map)))) + +(defun mu4e--mark-check-target (target) + "Check if TARGET exists; if not, offer to create it." + (let ((fulltarget (mu4e-join-paths (mu4e-root-maildir) target))) + (if (not (mu4e-create-maildir-maybe fulltarget)) + (mu4e-error "Target dir %s does not exist " fulltarget) + target))) + +(defun mu4e-mark-execute-all (&optional no-confirmation) + "Execute the actions for all marked messages in this buffer. +After the actions have been executed successfully, the affected +messages are *hidden* from the current header list. Since the +headers are the result of a search, we cannot be certain that the +messages no longer match the current one - to get that +certainty, we need to rerun the search, but we don't want to do +that automatically, as it may be too slow and/or break the user's +flow. Therefore, we hide the message, which in practice seems to +work well. + +If NO-CONFIRMATION is non-nil, don't ask user for confirmation." + (interactive "P") + (mu4e--mark-in-context + (let* ((marknum (mu4e-mark-marks-num)) + (prompt (format "Are you sure you want to execute %d mark%s?" + marknum (if (> marknum 1) "s" "")))) + (if (zerop marknum) + (mu4e-warn "Nothing is marked") + (mu4e-mark-resolve-deferred-marks) + (when (or no-confirmation (y-or-n-p prompt)) + (maphash + (lambda (docid val) + (let* ((mark (car val)) (target (cdr val)) + (markdescr (assq mark mu4e-marks)) + (msg (save-excursion + (mu4e~headers-goto-docid docid) + (mu4e-message-at-point)))) + ;; note: whenever you do something with the message, + ;; it looses its N (new) flag + (if markdescr + (progn + (run-hook-with-args + 'mu4e-mark-execute-pre-hook mark msg) + (funcall (plist-get (cdr markdescr) :action) + docid msg target)) + (mu4e-error "Unrecognized mark %S" mark)))) + mu4e--mark-map)) + (mu4e-mark-unmark-all 'no-confirm) + (message nil))))) + +(defun mu4e-mark-unmark-all (&optional no-confirmation) + "Unmark all marked messages." + (interactive) + (mu4e--mark-in-context + (when (zerop (mu4e-mark-marks-num)) + (mu4e-warn "Nothing is marked")) + (let* ((marknum (hash-table-count mu4e--mark-map)) + (prompt (format "Are you sure you want to unmark %d message%s?" + marknum (if (> marknum 1) "s" "")))) + (when (or no-confirmation (y-or-n-p prompt)) + (maphash + (lambda (docid _val) + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-mark-set 'unmark)))) + mu4e--mark-map) + ;; in any case, clear the marks map + (mu4e--mark-clear))))) + +(defun mu4e-mark-docid-marked-p (docid) + "Is the given DOCID marked?" + (when (gethash docid mu4e--mark-map) t)) + +(defun mu4e-mark-marks-num () + "Return the number of mark-instances in the current buffer." + (mu4e--mark-in-context + (if mu4e--mark-map (hash-table-count mu4e--mark-map) 0))) + +(defun mu4e-mark-handle-when-leaving () + "Handle any mark-instances in the current buffer when leaving. +This is done according to the value of +`mu4e-headers-leave-behavior'. This function is to be called +before any further action (like searching, quitting the buffer) +is taken; returning t means \"take the following action\", return +nil means \"don't do anything\"." + (mu4e--mark-in-context + (let ((marknum (mu4e-mark-marks-num)) + (what mu4e-headers-leave-behavior)) + (unless (zerop marknum) ;; nothing to do? + (when (eq what 'ask) + (setq what (mu4e-read-option + (format "There are %d existing mark(s); should we: " + marknum) + '( ("apply marks" . apply) + ("ignore marks?" . ignore))))) + ;; we determined what to do... now do it + (when (eq what 'apply) + (mu4e-mark-execute-all t))))) + t) ;; return t for compat with `kill-buffer-query-functions + +;;; _ +(provide 'mu4e-mark) +;;; mu4e-mark.el ends here diff --git a/mu4e/mu4e-message.el b/mu4e/mu4e-message.el new file mode 100644 index 0000000..95f8aff --- /dev/null +++ b/mu4e/mu4e-message.el @@ -0,0 +1,247 @@ +;;; mu4e-message.el --- Working with mu4e-message plists -*- lexical-binding: t -*- + +;; Copyright (C) 2012-2022 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Functions to get data from mu4e-message plist structure + +;;; Code: + +(require 'mu4e-vars) +(require 'mu4e-contacts) +(require 'mu4e-window) +(require 'flow-fill) +(require 'shr) +(require 'pp) + +(declare-function mu4e-error "mu4e-helpers") +(declare-function mu4e-warn "mu4e-helpers") +(declare-function mu4e-personal-address-p "mu4e-contacts") +(declare-function mu4e-make-temp-file "mu4e-helpers") + +;;; Message fields + +(defsubst mu4e-message-field-raw (msg field) + "Retrieve FIELD from message plist MSG. + +See \"mu fields\" for the full list of field, in particular the +\"sexp\" column. + +Returns nil if the field does not exist. + +A message plist looks something like: +\(:docid 32461 + :from ((:name \"Nikola Tesla\" :email \"niko@example.com\")) + :to ((:name \"Thomas Edison\" :email \"tom@example.com\")) + :cc ((:name \"Rupert The Monkey\" :email \"rupert@example.com\")) + :subject \"RE: what about the 50K?\" + :date (20369 17624 0) + :size 4337 + :message-id \"238C8233AB82D81EE81AF0114E4E74@123213.mail.example.com\" + :path \"/home/tom/Maildir/INBOX/cur/133443243973_1.10027.atlas:2,S\" + :maildir \"/INBOX\" + :priority normal + :flags (seen) +\)). +Some notes on the format: +- The address fields are lists of plist (:name NAME :email EMAIL), + where the :name part can be absent. The `mu4e-contact-name' and + `mu4e-contact-email' accessors can be useful for this. +- The date is in format emacs uses in `current-time' +- Attachments are a list of elements with fields :index (the number of + the MIME-part), :name (the file name, if any), :mime-type (the + MIME-type, if any) and :size (the size in bytes, if any). +- Messages in the Headers view come from the database and do not have + :attachments, :body-txt or :body-html fields. Message in the + Message view use the actual message file, and do include these fields." + ;; after all this documentation, the spectacular implementation + (if msg + (plist-get msg field) + (mu4e-error "Message must be non-nil"))) + +(defsubst mu4e-message-field (msg field) + "Retrieve FIELD from message plist MSG. +Like `mu4e-message-field-nil', but will sanitize nil values: +- all string field except body-txt/body-html: nil -> \"\" +- numeric fields + dates : nil -> 0 +- all others : return the value +Thus, function will return nil for empty lists, non-existing body-txt +or body-html." + (let ((val (mu4e-message-field-raw msg field))) + (cond + (val + val) ;; non-nil -> just return it + ((member field '(:subject :message-id :path :maildir :in-reply-to)) + "") ;; string fields except body-txt, body-html: nil -> "" + ((member field '(:body-html :body-txt)) + val) + ((member field '(:docid :size)) + 0) ;; numeric type: nil -> 0 + (t + val)))) ;; otherwise, just return nil + +(defsubst mu4e-message-has-field (msg field) + "If MSG has a FIELD return t, nil otherwise." + (plist-member msg field)) + +(defsubst mu4e-message-at-point (&optional noerror) + "Get the message s-expression for the message at point. +Either the headers buffer or the view buffer, or nil if there is +no such message. If optional NOERROR is non-nil, do not raise an +error when there is no message at point." + (or (cond + ((eq major-mode 'mu4e-headers-mode) (get-text-property (point) 'msg)) + ((eq major-mode 'mu4e-view-mode) mu4e--view-message)) + (unless noerror (mu4e-warn "No message at point")))) + +(defsubst mu4e-message-field-at-point (field) + "Get the field FIELD from the message at point. +This is equivalent to: + (mu4e-message-field (mu4e-message-at-point) FIELD)." + (mu4e-message-field (mu4e-message-at-point) field)) + +(defun mu4e-message-contact-field-matches (msg cfield rx) + "Does MSG's contact-field CFIELD match regexp RX? +Check if any of the of the CFIELD in MSG matches RX. I.e. +anything in field CFIELD (either :to, :from, :cc or :bcc, or a +list of those) of msg MSG matches (with their name or e-mail +address) regular expressions RX. If there is a match, return +non-nil; otherwise return nil. RX can also be a list of regular +expressions, in which case any of those are tried for a match." + (cond + ((null cfield)) + ((listp cfield) + (seq-find (lambda (cf) (mu4e-message-contact-field-matches msg cf rx)) + cfield)) + ((listp rx) + ;; if rx is a list, try each one of them for a match + (seq-find + (lambda (a-rx) (mu4e-message-contact-field-matches msg cfield a-rx)) + rx)) + (t + ;; not a list, check the rx + (seq-find + (lambda (ct) + (let ((name (mu4e-contact-name ct)) + (email (mu4e-contact-email ct)) + ;; the 'rx' may be some `/rx/` from mu4e-personal-addresses; + ;; so let's detect and extract in that case. + (rx (if (string-match-p "^\\(.*\\)/$" rx) + (substring rx 1 -1) rx))) + (or + (and name (string-match rx name)) + (and email (string-match rx email))))) + (mu4e-message-field msg cfield))))) + +(defun mu4e-message-contact-field-matches-me (msg cfield) + "Does contact-field CFIELD in MSG match me? +Checks whether any +of the of the contacts in field CFIELD (either :to, :from, :cc or +:bcc) of msg MSG matches *me*, that is, any of the addresses for +which `mu4e-personal-address-p' return t. Returns the contact +cell that matched, or nil." + (seq-find (lambda (cell) + (mu4e-personal-address-p (mu4e-contact-email cell))) + (mu4e-message-field msg cfield))) + +(defun mu4e-message-sent-by-me (msg) + "Is this MSG (to be) sent by me? +Checks if the from field matches user's personal addresses." + (mu4e-message-contact-field-matches-me msg :from)) + +(defun mu4e-message-personal-p (msg) + "Does MSG have user's personal address? +In any of the contact + fields?" + (seq-some + (lambda (field) + (mu4e-message-contact-field-matches-me msg field)) + '(:from :to :cc :bcc))) + +(defsubst mu4e-message-part-field (msgpart field) + "Get some FIELD from MSGPART. +A part would look something like: + (:index 2 :name \"photo.jpg\" :mime-type \"image/jpeg\" :size 147331)." + (plist-get msgpart field)) + +;; backward compatibility ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(defalias 'mu4e-msg-field 'mu4e-message-field) + +(defun mu4e-field-at-point (field) + "Get FIELD for the message at point. +Either in the headers buffer or the view buffer. Field is a +symbol, see `mu4e-header-info'." + (plist-get (mu4e-message-at-point) field)) + +(defun mu4e-message-readable-path (&optional msg) + "Get a readable path to MSG or raise an error. +If MSG is nil, use `mu4e-message-at-point'." + (let ((path (plist-get (or msg (mu4e-message-at-point)) :path))) + (unless (file-readable-p path) + (mu4e-error "No readable message at %s; database outdated?" path)) + path)) + +(defun mu4e-copy-message-path () + "Copy the message-path of message at point to the kill ring." + (interactive) + (let ((path (mu4e-message-field-at-point :path))) + (kill-new path) + (mu4e-message "Saved '%s' to kill-ring" path))) + +(defun mu4e-sexp-at-point () + "Show or hide the s-expression for the message-at-point, if any." + (interactive) + (if-let ((win (get-buffer-window mu4e--sexp-buffer-name))) + (delete-window win) + (when-let ((msg (mu4e-message-at-point 'noerror))) + (when (buffer-live-p mu4e--sexp-buffer-name) + (kill-buffer mu4e--sexp-buffer-name)) + (with-current-buffer-window + (get-buffer-create mu4e--sexp-buffer-name) nil nil + (if (fboundp 'lisp-data-mode) + (lisp-data-mode) + (lisp-mode)) + (insert (pp-to-string msg)) + (font-lock-ensure) + ;; add basic `quit-window' bindings + (view-mode 1))))) + +(declare-function mu4e--decoded-message "mu4e-compose") + +(defun mu4e-fetch-field (msg hdr &optional first) + "Find the value for an arbitrary header field HDR from MSG. + +If the header appears multiple times, the field values are +concatenated, unless FIRST is non-nil, in which case only the +first value is returned. See `message-field-value' and +`nessage-fetch-field' for details. + +Note: this loads the full message file such that any available +message header can be used. If the header is part of the MSG +plist, it is much more efficient to get the information from that +plist." + (with-temp-buffer + (insert (mu4e--decoded-message msg 'headers-only)) + (message-field-value hdr first))) +;;; +(provide 'mu4e-message) +;;; mu4e-message.el ends here diff --git a/mu4e/mu4e-mime-parts.el b/mu4e/mu4e-mime-parts.el new file mode 100644 index 0000000..73a1742 --- /dev/null +++ b/mu4e/mu4e-mime-parts.el @@ -0,0 +1,486 @@ +;;; mu4e-mime-parts.el --- Dealing with MIME-parts & URLs -*- lexical-binding: t -*- + +;; Copyright (C) 2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Implements functions and variables for dealing with MIME-parts and URLs. + + +;;; TODO: +;; [~] mime part candidate sorting -> is his even possible generally? +;; [ ] URL support + +;;; Code: + + +(require 'mu4e-vars) +(require 'mu4e-folders) +(require 'gnus-art) + + + +(defcustom mu4e-view-open-program + (pcase system-type + ('darwin "open") + ('cygwin "cygstart") + (_ "xdg-open")) + "Tool to open the correct program for a given file or MIME-type. +May also be a function of a single argument, the file to be +opened. + +In the function-valued case a likely candidate is +`mailcap-view-file' although note that there was an Emacs bug up +to Emacs 29 which prevented opening a file if `mailcap-mime-data' +specified a function as viewer." + :type '(choice string function) + :group 'mu4e-view) + + +;; remember the mime-handles, so we can clean them up when +;; we quit this buffer. +(defvar-local mu4e~gnus-article-mime-handles nil) +(put 'mu4e~gnus-article-mime-handles 'permanent-local t) + +(defun mu4e--view-kill-mime-handles () + "Kill cached MIME-handles, if any." + (when mu4e~gnus-article-mime-handles + (mm-destroy-parts mu4e~gnus-article-mime-handles) + (setq mu4e~gnus-article-mime-handles nil))) + + +;;; MIME-parts +(defvar-local mu4e--view-mime-parts nil + "Cached MIME parts for this message.") + + +(defun mu4e-view-mime-parts() + "Get the list of MIME parts for this message. +The list is a list of plists, one for each MIME-part. + +The plists have the properties: + + :part-index : Gnus index number + :mime-type : MIME-type (string) or nil + :encoding : Content encoding (string) or nil + :disposition : Content disposition (attachment\" or inline\") or nil + :filename : The file name if it has one, or an invented one + otherwise + +There are some internal fields as well, e.g. ; subject to change: + + :target-dir : Target directory for saving + :attachment-like : When it has a filename, we can save it + :handle : Gnus handle." + (or mu4e--view-mime-parts + (setq + mu4e--view-mime-parts + (let ((parts) (indices)) + (save-excursion + (goto-char (point-min)) + (while (not (eobp)) + (when-let ((part (get-text-property (point) 'gnus-data)) + (index (get-text-property (point) 'gnus-part))) + (when (and part (numberp index) (not (member index indices))) + (let* ((disp (mm-handle-disposition part)) + (fname (mm-handle-filename part)) + (mime-type (mm-handle-media-type part)) + (info + `(:part-index ,index + :mime-type ,mime-type + :encoding ,(mm-handle-encoding part) + :disposition ,(car-safe disp) + + ;; if there's no file-name, invent one + ;; XXX perhaps guess extension based on mime-type + :filename ,(or fname + (format "mime-part-%02d" index)) + + ;; below are internal + + :target-dir ,(mu4e-determine-attachment-dir + fname mime-type) + ;; 'attachment-like' just means it has its own + ;; filename an we thus we can save it through + ;; `mu4e-view-save-attachments', even if it has an + ;; 'inline' disposition. + :attachment-like ,(if fname t nil) + :handle ,part))) + (push index indices) + (push info parts)))) + (goto-char (or (next-single-property-change (point) 'gnus-part) + (point-max))))) + ;; sort by the GNU's part-index, so the order is the same as + ;; in the message on screen + (seq-sort (lambda (p1 p2) (< (plist-get p1 :part-index) + (plist-get p2 :part-index))) parts))))) + +;; https://emacs.stackexchange.com/questions/74547/completing-read-search-also-in-annotationsxc + +(defun mu4e--uniqify-file-name (fname) + "Return a non-yet-existing filename based on FNAME. +If FNAME does not yet exist, return it unchanged. +Otherwise, return a file with a unique number appended to the base-name." + (let ((num 1) (orig-name fname)) + (while (file-exists-p fname) + (setq fname (format "%s(%d)%s%s" + (file-name-sans-extension orig-name) + num + (if (file-name-extension orig-name) "." "") + (file-name-extension orig-name))) + (cl-incf num))) + fname) + +(defvar mu4e--completions-table nil) + +(defun mu4e-view-complete-all () + "Pick all current candidates." + (interactive) + (if (bound-and-true-p helm-mode) + (mu4e-warn "Not supported with helm") + (when mu4e--completions-table + (insert (string-join + (seq-map #'car mu4e--completions-table) ", "))))) + +(defvar mu4e-view-completion-minor-mode-map + (let ((map (make-sparse-keymap))) + (define-key map (kbd "C-c C-a") #'mu4e-view-complete-all) + ;; XXX perhaps a binding for clearing all? + map) + "Keybindings for mu4e-view completion.") + +(define-minor-mode mu4e-view-completion-minor-mode + "Minor-mode for completing mu4e mime parts." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap mu4e-view-completion-minor-mode-map) + +(defun mu4e--part-annotation (candidate part type longest-filename) + "Calculate the annotation candidates as per +`:annotation-function' (see `completion-extra-properties') + +CANDIDATE is the value to annotate. + +PART is the matching MIME-part for the annotation, (as per +`mu4e-view-mime-part'). + +TYPE is the of what to annotate, a symbol, either ATTACHMENT or +MIME-PART. + +LONGEST-FILENAME is the length of the longest filename; this +information' is used for alignment." + (let* ((filename (propertize (or (plist-get part :filename) "") + 'face 'mu4e-header-key-face)) + (mimetype (propertize (or (plist-get part :mime-type) "") + 'face 'mu4e-header-value-face)) + (target (propertize (or (plist-get part :target-dir) "") + 'face 'mu4e-system-face))) + + ;; Sadly, we need too align by hand; this makes some assumptions + ;; such a mono-type font and enough space in the minibuffer; and + ;; mixing values and representation; ideally Emacs would allow + ;; just take some columns and align them (since it knows the display + ;; details). + + (pcase type + ('attachment + ;; in case we're annotating an attachment, the filename is + ;; the candidate (completion), so we don't need it in the + ;; the annotation. We just need to but some space at beginning + ;; for alignment + (concat + (make-string (- (+ longest-filename 2) + (length (format "%s" candidate))) ?\s) + (format "%20s" mimetype) + " " + (format "%s" (concat "-> " target)))) + ('mime-part + ;; when we're annotating a mime-part, the candidate is just a number, + ;; and the filename is part of the annotation. + (concat + " " + filename + (make-string (- (+ longest-filename 2) + (length filename)) ?\s) + (format "%20s" mimetype) + " " + (format "%s" (concat "-> " target)))) + (_ (mu4e-error "Unsupported annotation type %s" type))))) + + +(defvar helm-comp-read-use-marked) +(defun mu4e--completing-read-real (prompt candidates multi) + "Call the appropriate completion-read function. +- PROMPT is a string informing the user what to complete +- CANDIDATES is an alist of candidates of the form + (id . part) +- MULTI if t, allow for completing _multiple_ candidates." + (cond + ((bound-and-true-p helm-mode) + ;; tweaks for "helm"; it's not nice to have to special-case for + ;; completion frameworks, but this has been supported for while. + ;; basically, with helm, helm-comp-read-use-marked + completing-read + ;; is preferred over completing-read-multiple + (let ((helm-comp-read-use-marked t)) + (completing-read prompt candidates))) + (multi + (completing-read-multiple prompt candidates)) + (t + (completing-read prompt candidates)))) + +(defun mu4e--completing-read (prompt candidates type &optional multi) + "Read the part-id of some MIME-type in this message. + +Presents the user with completions for the MIME-parts in +the current message. + +- PROMPT is a string informing the user what to complete +- CANDIDATES is an alist of candidates of the form + (id . part) +- TYPE is the annotation type to uses as per `mu4e--part-annotation'. +Optionally, +- MULTI if t, allow for completing _multiple_ candidates." + (cl-assert candidates) + (let* ((longest-filename (seq-max + (seq-map (lambda (c) + (length (plist-get (cdr c) :filename))) + candidates))) + (annotation-func (lambda (candidate) + (mu4e--part-annotation candidate + (cdr-safe + (assoc candidate candidates)) + type longest-filename))) + (completion-extra-properties + `(;; :affixation-function requires emacs 28 + :annotation-function ,annotation-func + :exit-function (lambda (_a _b) (setq mu4e--completions-table nil))))) + (setq mu4e--completions-table candidates) + (minibuffer-with-setup-hook + (lambda () + (mu4e-view-completion-minor-mode)) + (mu4e--completing-read-real prompt candidates multi)))) + +(defun mu4e-view-save-attachments (&optional ask-dir) + "Save files from the current view buffer. +This applies to all MIME-parts that are \"attachment-like\" (have a filename), +regardless of their disposition. + +With ASK-DIR is non-nil, user can specify the target-directory; otherwise +one is determined using `mu4e-attachment-dir'." + (interactive "P") + (let* ((parts (mu4e-view-mime-parts)) + (candidates (seq-map + (lambda (fpart) + (cons ;; (filename . annotation) + (plist-get fpart :filename) + fpart)) + (seq-filter + (lambda (part) (plist-get part :attachment-like)) + parts))) + (candidates (or candidates + (mu4e-warn "No attachments for this message"))) + (files (mu4e--completing-read "Save file(s): " candidates + 'attachment 'multi)) + (custom-dir (when ask-dir (read-directory-name + "Save to directory: ")))) + ;; we have determined what files to save, and where. + (seq-do (lambda (fname) + (let* ((part (cdr (assoc fname candidates))) + (path (mu4e--uniqify-file-name + (mu4e-join-paths + (or custom-dir (plist-get part :target-dir)) + (plist-get part :filename))))) + (mm-save-part-to-file (plist-get part :handle) path))) + files))) + +(defvar mu4e-view-mime-part-actions + '( + ;; + ;; some basic ones + ;; + + ;; save MIME-part to a file + (:name "save" :handler gnus-article-save-part :receives index) + ;; pipe MIME-part to some arbitrary shell command + (:name "|pipe" :handler gnus-article-pipe-part :receives index) + ;; open with the default handler, if any + (:name "open" :handler mu4e--view-open-file :receives temp) + ;; open with some custom file. + (:name "wopen-with" :handler (lambda (file)(mu4e--view-open-file file t)) + :receives temp) + + ;; + ;; some more examples + ;; + + ;; import GPG key + (:name "gpg" :handler epa-import-keys :receives temp) + ;; open in this emacs instance; tries to use the attachment name, + ;; so emacs can use specific modes etc. + (:name "emacs" :handler find-file-read-only :receives temp) + ;; open in this emacs instance, "raw" + (:name "raw" :handler (lambda (str) + (let ((tmpbuf + (get-buffer-create " *mu4e-raw-mime*"))) + (with-current-buffer tmpbuf + (insert str) + (view-mode) + (goto-char (point-min))) + (display-buffer tmpbuf))) :receives pipe)) + + "Specifies actions for MIME-parts. + +Each of the actions is a plist with keys +`(:name <name> ;; name of the action; shortcut is first letter of name + + :handler ;; one of: + ;; - a function receiving the index/temp/pipe + ;; - a string, which is taken as a shell command + + :receives ;; a symbol specifying what the handler receives + ;; - index: the index number of the mime part (default) + ;; - temp: the full path to the mime part in a + ;; temporary file, which is deleted immediately + ;; after the handler returns + ;; - pipe: the attachment is piped to some shell command + ;; or as a string parameter to a function +).") + + +(defun mu4e--view-mime-part-to-temp-file (handle) + "Write MIME-part HANDLE to a temporary file and return the file name. +The filename is deduced from the MIME-part's filename, or +otherwise random; the result is placed in a temporary directory +with a unique name. Returns the full path for the file created. +The directory and file are self-destructed." + (let* ((tmpdir (make-temp-file "mu4e-temp-" t)) + (fname (mm-handle-filename handle)) + (fname (and fname + (gnus-map-function mm-file-name-rewrite-functions + (file-name-nondirectory fname)))) + (fname (if fname + (concat tmpdir "/" (replace-regexp-in-string "/" "-" fname)) + (let ((temporary-file-directory tmpdir)) + (make-temp-file "mimepart"))))) + (mm-save-part-to-file handle fname) + (run-at-time "30 sec" nil + (lambda () (ignore-errors (delete-directory tmpdir t)))) + fname)) + +(defun mu4e--view-open-file (file &optional force-ask) + "Open FILE with default handler, if any. +Otherwise, or if FORCE-ASK is set, ask user for the program to +open with." + (if (and (not force-ask) + (functionp mu4e-view-open-program)) + (funcall mu4e-view-open-program file) + (let ((opener + (or (and (not force-ask) mu4e-view-open-program + (executable-find mu4e-view-open-program)) + (read-shell-command "Open MIME-part with: ")))) + (call-process opener nil 0 nil file)))) + +(defun mu4e-view-mime-part-action (&optional n) + "Apply some action to MIME-part N in the current message. +If N is not specified, ask for it. For instance, '3 A o' opens +the third MIME-part." + ;; (interactive + ;; (list (read-number "Number of MIME-part: "))) + (interactive) + (let* ((parts (mu4e-view-mime-parts)) + (candidates (seq-map + (lambda (part) + (cons (number-to-string + (plist-get part :part-index)) part)) + parts)) + (candidates (or candidates + (mu4e-warn "No MIME-parts for this message"))) + (ids (seq-map #'string-to-number + (if n (list (number-to-string n)) + (mu4e--completing-read "MIME-part(s) to operate on: " + candidates + 'mime-part 'multi)))) + (options + (mapcar (lambda (action) `(,(plist-get action :name) . ,action)) + mu4e-view-mime-part-actions)) + (action + (or (and options (mu4e-read-option "Action: " options)) + (mu4e-error "No such action"))) + (handler + (or (plist-get action :handler) + (mu4e-error "No :handler item found for action %S" action))) + (receives + (or (plist-get action :receives) + (mu4e-error "No :receives item found for action %S" action)))) + + ;; Apply the action to all selected MIME-parts + (seq-do (lambda (id) + (cl-assert (numberp id)) + (let* ((part (or (cdr-safe (assoc (number-to-string id) candidates)) + (mu4e-error "No part found for id %s" id))) + (handle (plist-get part :handle))) + (save-excursion + (cond + ((functionp handler) + (cond + ((eq receives 'index) (funcall handler id)) + ((eq receives 'pipe) + (funcall handler (mm-with-unibyte-buffer + (mm-insert-part handle) + (buffer-string)))) + ((eq receives 'temp) + (funcall handler + (mu4e--view-mime-part-to-temp-file handle))) + (t (mu4e-error "Invalid :receive for %S" action)))) + ((stringp handler) + (cond + ((eq receives 'index) + (shell-command + (concat handler " " (shell-quote-argument id)))) + ((eq receives 'pipe) + (progn + (mm-pipe-part handle handler))) + ((eq receives 'temp) + (shell-command + (shell-command + (concat + handler " " + (shell-quote-argument + (mu4e--view-mime-part-to-temp-file handle)))))) + (t (mu4e-error "Invalid action %S" action)))))))) + ids))) + +(defun mu4e-process-file-through-pipe (path pipecmd) + "Process file at PATH through a pipe with PIPECMD." + (let ((buf (get-buffer-create "*mu4e-output"))) + (with-current-buffer buf + (let ((inhibit-read-only t)) + (erase-buffer) + (call-process-shell-command pipecmd path t t) + (view-mode))) + (display-buffer buf))) + + + +(provide 'mu4e-mime-parts) +;;; mu4e-mime-parts.el ends here diff --git a/mu4e/mu4e-modeline.el b/mu4e/mu4e-modeline.el new file mode 100644 index 0000000..f1e668d --- /dev/null +++ b/mu4e/mu4e-modeline.el @@ -0,0 +1,141 @@ +;;; mu4e-modeline.el --- Modeline for mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; This file contains functionality for putting mu4e-related information in the +;; Emacs modeline, both buffer-specific and globally. + +;;; Code: + +(require 'cl-lib) + + +(defcustom mu4e-modeline-max-width 42 + "Determines the maximum length of the local modeline string. +If the string exceeds this limit, it will be truncated to fit. + +Note: this only affects the local modeline items (such as the context, +the search properties and the last query), not the global items +(such as the favorite bookmark results)." + :type 'integer + :group 'mu4e-modeline) + +(defcustom mu4e-modeline-prefer-bookmark-name t + "Show bookmark name rather than query in modeline. + +If non-nil, if the current search query matches some bookmark, +display the bookmark name rather than the query." + :type 'boolean + :group 'mu4e-modeline) + +(defcustom mu4e-modeline-show-global t + "Whether to populate global modeline segments. + +If non-nil, show both buffer-specific and global modeline items, +otherwise only present buffer-specific information." + :type 'boolean + :group 'mu4e-modeline) + +(defvar-local mu4e--modeline-buffer-items nil + "List of buffer-local items for the mu4e modeline. +Each element is function that evaluates to a string.") + +(defvar mu4e--modeline-global-items nil + "List of items for the global modeline. +Each element is function that evaluates to a string.") + +(defun mu4e--modeline-register (func &optional global) + "Register FUNC for calculating some mu4e modeline part. +If GLOBAL is non-nil, add to the global-modeline; otherwise use +the buffer-local one." + (add-to-list + (if global + 'mu4e--modeline-global-items + 'mu4e--modeline-buffer-items) + func 'append)) + +(defun mu4e--modeline-quote-and-truncate (str) + "Quote STR to be used literally in the modeline. +The string is truncaed to fit if its length exceeds +`mu4e-modeline-max-width'." + (replace-regexp-in-string + "%" "%%" + (truncate-string-to-width str mu4e-modeline-max-width 0 nil t))) + +(defvar mu4e--modeline-item nil + "Mu4e item for the global-mode-line.") + +(defvar mu4e--modeline-global-string-cached nil + "Cached version of the _global_ modeline string. +Note that we don't cache the local parts, so that the modeline +gets updated when we leave the buffer from which the local parts +originate.") + +(defun mu4e--modeline-string () + "Get the current mu4e modeline string." + (let* ((collect + (lambda (lst) + (mapconcat + (lambda (func) (or (funcall func) "")) lst " "))) + (global-string ;; global string is _cached_ as it may be expensive. + (and + mu4e-modeline-show-global + (or mu4e--modeline-global-string-cached + (setq mu4e--modeline-global-string-cached + (funcall collect mu4e--modeline-global-items)))))) + (concat + ;; (local) buffer items are _not_ cached, so they'll get update + ;; automatically when leaving the buffer. + (mu4e--modeline-quote-and-truncate + (funcall collect mu4e--modeline-buffer-items)) + (and global-string " ") + global-string))) + +(define-minor-mode mu4e-modeline-mode + "Minor mode for showing mu4e information on the modeline." + ;; This is a bit special 'global' mode, since it consists of both + ;; buffer-specific parts (mu4e--modeline-buffer-items) and global items + ;; (mu4e--modeline-global-items). + :global t + :group 'mu4e + :lighter nil + (if mu4e-modeline-mode + (progn + (setq mu4e--modeline-item '(:eval (mu4e--modeline-string))) + (add-to-list 'global-mode-string mu4e--modeline-item) + (mu4e--modeline-update)) + (progn + (setq global-mode-string + (seq-remove (lambda (item) (equal item mu4e--modeline-item)) + global-mode-string))) + (force-mode-line-update))) + +(defun mu4e--modeline-update () + "Recalculate and force-update the modeline." + (when mu4e-modeline-mode + (setq mu4e--modeline-global-string-cached nil) + (force-mode-line-update))) + +(provide 'mu4e-modeline) + +;;; mu4e-modeline.el ends here diff --git a/mu4e/mu4e-notification.el b/mu4e/mu4e-notification.el new file mode 100644 index 0000000..9ad9638 --- /dev/null +++ b/mu4e/mu4e-notification.el @@ -0,0 +1,99 @@ +;;; mu4e-notification.el --- Showing mail notifications -*- lexical-binding: t-*- +;; +;; Copyright (C) 1996-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;;; Commentary: +;;; Generic support for showing new-mail notifications. + +;;; Code: + +(require 'mu4e-query-items) +(require 'mu4e-bookmarks) + +;; for emacs' built-in desktop notifications to work, we need +;; dbus +(when (featurep 'dbus) + (require 'notifications)) + +(defcustom mu4e-notification-filter #'mu4e--default-notification-filter + "Function for determining if a notification is to be emitted. + +If this is the case, the function should return non-nil. +The function must accept an optional single parameter, unused for +now." + :type 'function + :group 'mu4e-notification) + +(defcustom mu4e-notification-function + #'mu4e--default-notification-function + "Function to emit a notification. + +The function is invoked when we need to emit a new-mail +notification in some system-specific way. The function is invoked +when the query-items have been updated and +`mu4e-notification-filter' returns non-nil. + +The function must accept an optional single parameter, unused for +now." + :type 'function + :group 'mu4e-notification) + +(defvar mu4e--notification-id nil + "The last notification id, so we can replace it.") + +(defun mu4e--default-notification-filter (&optional _) + "Return t if a notification should be shown. + +This default implementation does so when the number of unread +messages changed since the last notification and it is greater +than zero." + (when-let* ((fav (mu4e-bookmark-favorite)) + (delta-unread (plist-get fav :delta-unread))) + (when (and (> delta-unread 0) + (not (= delta-unread mu4e--last-delta-unread))) + (setq mu4e--last-delta-unread delta-unread) ;; update + t ;; do show notification + ))) + +(defun mu4e--default-notification-function (&optional _) + "Default function for handling notifications. +The default implementation uses emacs' built-in dbus-notification +support." + (when-let* ((fav (mu4e-bookmark-favorite)) + (title "mu4e found new mail") + (delta-unread (or (plist-get fav :delta-unread) 0)) + (body (format "%d new message%s in %s" + delta-unread + (if (= delta-unread 1) "" "s") + (plist-get fav :name)))) + (cond + ((fboundp 'do-applescript) + (do-applescript + (format "display notification %S with title %S" body title))) + ((fboundp 'notifications-notify) + ;; notifications available + (setq mu4e--notification-id + (notifications-notify + :title title + :body body + :app-name "mu4e@emacs" + :replaces-id mu4e--notification-id + ;; a custom mu4e icon would be nice... + ;; :app-icon (ignore-errors + ;; (image-search-load-path + ;; "gnus/gnus.png")) + :actions '("Show" "Favorite bookmark" + "default" "Favorite bookmark") + :on-action (lambda (_1 _2) (mu4e-jump-to-favorite))))) + ;; ... TBI: other notifications ... + (t ;; last resort + (mu4e-message "%s: %s" title body))))) + +(defun mu4e--notification () + "Function called when the query items have been updated." + (when (and (funcall mu4e-notification-filter) + (functionp mu4e-notification-function)) + (funcall mu4e-notification-function))) + +(provide 'mu4e-notification) +;;; mu4e-notification.el ends here diff --git a/mu4e/mu4e-obsolete.el b/mu4e/mu4e-obsolete.el new file mode 100644 index 0000000..c040e1a --- /dev/null +++ b/mu4e/mu4e-obsolete.el @@ -0,0 +1,283 @@ +;;; mu4e-obsolete.el --- Obsolete things -*- lexical-binding: t -*- + +;; Copyright (C) 2022-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Obsolete variable & function aliases go here, so we don't clutter up the +;; code. + +;;; Code: + + +;; mu4e-draft/compose + +(make-obsolete-variable 'mu4e-reply-to-address + 'mu4e-compose-reply-to-address + "v0.9.9") + +(make-obsolete-variable 'mu4e-auto-retrieve-keys "no longer used." "1.3.1") + +(make-obsolete-variable 'mu4e-compose-func "no longer used" "1.11.26") + +(make-obsolete-variable 'mu4e-compose-crypto-reply-encrypted-policy "The use of the + 'mu4e-compose-crypto-reply-encrypted-policy' variable is deprecated. + 'mu4e-compose-crypto-policy' should be used instead" "2020-03-06") + +(make-obsolete-variable 'mu4e-compose-crypto-reply-plain-policy "The use of the + 'mu4e-compose-crypto-reply-plain-policy' variable is deprecated. + 'mu4e-compose-crypto-policy' should be used instead" + "2020-03-06") + +(make-obsolete-variable 'mu4e-compose-crypto-reply-policy "The use of the + 'mu4e-compose-crypto-reply-policy' variable is deprecated. + 'mu4e-compose-crypto-reply-plain-policy' and + 'mu4e-compose-crypto-reply-encrypted-policy' should be used instead" + "2017-09-02") + +(make-obsolete-variable 'mu4e-compose-auto-include-date + "This is done unconditionally now" "1.3.5") + +(make-obsolete-variable 'mu4e-compose-signature-auto-include + "Usage message-signature directly" "1.11.22") + +(define-obsolete-variable-alias + 'mu4e-compose-signature 'message-signature "1.11.22") +(define-obsolete-variable-alias + 'mu4e-compose-cite-function 'message-cite-function "1.11.22") +(define-obsolete-variable-alias + 'mu4e-compose-in-new-frame 'mu4e-compose-switch "1.11.22") + +(define-obsolete-variable-alias 'mu4e-compose-hidden-headers + 'mu4e-draft-hidden-headers "1.12.5") + + +;; mu4e-message + +(make-obsolete-variable 'mu4e-html2text-command "No longer in use" "1.7.0") +(make-obsolete-variable 'mu4e-view-prefer-html "No longer in use" "1.7.0") +(make-obsolete-variable 'mu4e-view-html-plaintext-ratio-heuristic + "No longer in use" "1.7.0") +(make-obsolete-variable 'mu4e-message-body-rewrite-functions + "No longer in use" "1.7.0") +;;; Html2Text +(make-obsolete 'mu4e-shr2text "No longer in use" "1.7.0") + + + +;; old message view +(make-obsolete-variable 'mu4e-view-show-addresses + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-wrap-lines nil "0.9.9-dev7") +(make-obsolete-variable 'mu4e-view-hide-cited nil "0.9.9-dev7") +(make-obsolete-variable 'mu4e-view-date-format + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-image-max-width + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-image-max-height + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-save-multiple-attachments-without-asking + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-attachment-assoc + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-attachment-actions + "See mu4e-view-mime-part-actions" "1.7.0") +(make-obsolete-variable 'mu4e-view-header-field-keymap + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-header-field-keymap + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-contacts-header-keymap + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-attachments-header-keymap + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-imagemagick-identify nil "1.7.0") +(make-obsolete-variable 'mu4e-view-show-images + "No longer used" "1.7.0") +(make-obsolete-variable 'mu4e-view-gnus "Old view is gone" "1.7.0") +(make-obsolete-variable 'mu4e-view-use-gnus "Gnus view is the default" "1.5.10") + +(make-obsolete-variable 'mu4e-cited-regexp "No longer used" "1.7.0") + +(define-obsolete-variable-alias 'mu4e-view-blocked-images 'gnus-blocked-images + "1.5.12") +(define-obsolete-variable-alias 'mu4e-view-inhibit-images 'gnus-inhibit-images + "1.5.12") + +(define-obsolete-variable-alias 'mu4e-after-view-message-hook + 'mu4e-view-rendered-hook "1.9.7") + + +;; mu4e-org +(define-obsolete-function-alias 'org-mu4e-open 'mu4e-org-open "1.3.6") +(define-obsolete-function-alias 'org-mu4e-store-and-capture + 'mu4e-org-store-and-capture "1.3.6") + + +;; mu4e-search +(define-obsolete-variable-alias 'mu4e-headers-results-limit + 'mu4e-search-results-limit "1.7.0") +(define-obsolete-variable-alias 'mu4e-headers-full-search + 'mu4e-search-full "1.7.0") +(define-obsolete-variable-alias 'mu4e-headers-show-threads + 'mu4e-search-threads "1.7.0") +(define-obsolete-variable-alias + 'mu4e-headers-search-bookmark-hook + 'mu4e-search-bookmark-hook "1.7.0") +(define-obsolete-variable-alias 'mu4e-headers-search-hook + 'mu4e-search-hook "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search 'mu4e-search "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search-edit + 'mu4e-search-edit "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search-bookmark + 'mu4e-search-bookmark "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search-bookmark-edit + 'mu4e-search-bookmark-edit "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search-narrow + 'mu4e-search-narrow "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-rerun-search + 'mu4e-search-rerun "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-query-next + 'mu4e-search-next "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-query-prev + 'mu4e-search-prev "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-forget-queries + 'mu4e-search-forget "1.7.0") +(define-obsolete-function-alias 'mu4e-read-query + 'mu4e-search-read-query "1.7.0") + +(make-obsolete-variable 'mu4e-display-update-status-in-modeline + "No longer used" "1.9.11") + +;; mu4e-headers +(make-obsolete-variable 'mu4e-headers-field-properties-function + "not used" "1.6.1") + +(define-obsolete-function-alias 'mu4e-headers-toggle-setting + 'mu4e-headers-toggle-property "1.9.5") +(define-obsolete-function-alias 'mu4e-headers-toggle-threading + 'mu4e-headers-toggle-property "1.9.5") +(define-obsolete-function-alias 'mu4e-headers-toggle-full-search + 'mu4e-headers-toggle-property "1.9.5") +(define-obsolete-function-alias 'mu4e-headers-toggle-include-related + 'mu4e-headers-toggle-property "1.9.5") +(define-obsolete-function-alias 'mu4e-headers-toggle-skip-duplicates + 'mu4e-headers-toggle-property "1.9.5") + +(define-obsolete-function-alias 'mu4e-headers-change-sorting + 'mu4e-search-change-sorting "1.9.11") +(define-obsolete-function-alias 'mu4e-headers-toggle-property + 'mu4e-search-toggle-property "1.9.11") + +(define-obsolete-variable-alias 'mu4e-headers-include-related + 'mu4e-search-include-related "1.9.11") +(define-obsolete-variable-alias 'mu4e-headers-skip-duplicates + 'mu4e-search-skip-duplicates "1.9.11") +(define-obsolete-variable-alias 'mu4e-headers-sort-field + 'mu4e-search-sort-field "1.9.11") +(define-obsolete-variable-alias 'mu4e-headers-sort-direction + 'mu4e-search-sort-direction "1.9.11") + +(define-obsolete-variable-alias 'mu4e-headers-hide-predicate + 'mu4e-search-hide-predicate "1.9.11") +(define-obsolete-variable-alias 'mu4e-headers-hide-enabled + 'mu4e-search-hide-enabled "1.9.11") + +(define-obsolete-variable-alias 'mu4e-headers-threaded-label + 'mu4e-search-threaded-label "1.9.12") +(define-obsolete-variable-alias 'mu4e-headers-full-label + 'mu4e-search-full-label "1.9.12") +(define-obsolete-variable-alias 'mu4e-headers-related-label + 'mu4e-search-related-label "1.9.12") +(define-obsolete-variable-alias 'mu4e-headers-skip-duplicates-label + 'mu4e-search-skip-duplicates-label "1.9.12") +(define-obsolete-variable-alias 'mu4e-headers-hide-label + 'mu4e-search-hide-label "1.9.12") +;; by exception, add alias for internal func +(define-obsolete-function-alias 'mu4e~headers-jump-to-maildir + 'mu4e-search-maildir "1.9.13") + + +;; mu4e-main +(define-obsolete-variable-alias + 'mu4e-main-buffer-hide-personal-addresses + 'mu4e-main-hide-personal-addresses "1.5.7") + + +;; mu4e-server + +(make-obsolete-variable + 'mu4e-maildir + "determined by server; see `mu4e-root-maildir'." "1.3.8") + +(make-obsolete-variable 'mu4e-header-func "mu4e-headers-append-func" "1.7.4") +(make-obsolete-variable 'mu4e-temp-func "No longer used" "1.7.0") + + +;; mu4e-update +(define-obsolete-function-alias 'mu4e-interrupt-update-mail + 'mu4e-kill-update-mail "1.0-alpha0") + +;; mu4e-helpers +(define-obsolete-function-alias 'mu4e-quote-for-modeline + 'mu4e--modeline-quote-and-truncate "1.9.16") + +;; mu4e-folder +(make-obsolete-variable 'mu4e-cache-maildir-list "No longer used" "1.11.15") + +;; mu4e-contacts + +(define-obsolete-function-alias 'mu4e-user-mail-address-p + 'mu4e-personal-address-p "1.5.5") + +;; don't use the older vars anymore +(make-obsolete-variable 'mu4e-user-mail-address-regexp + 'mu4e-user-mail-address-list "0.9.9.x") +(make-obsolete-variable 'mu4e-my-email-addresses + 'mu4e-user-mail-address-list "0.9.9.x") +(make-obsolete-variable 'mu4e-user-mail-address-list + "determined by server; see `mu4e-personal-addresses'." + "1.3.8") +(make-obsolete-variable 'mu4e-contact-rewrite-function + "mu4e-contact-process-function (see docstring)" + "1.3.2") +(make-obsolete-variable 'mu4e-compose-complete-ignore-address-regexp + "mu4e-contact-process-function (see docstring)" + "1.3.2") + +(make-obsolete-variable 'mu4e-compose-reply-recipients + "use mu4e-compose-reply / mu4e-compose-wide-reply" + "1.11.23") +(make-obsolete-variable 'mu4e-compose-reply-ignore-address + "see: message-prune-recipient-rules" "1.11.23") + +;; this is only a _rough_ +(make-obsolete-variable 'mu4e-compose-dont-reply-to-self + "message-dont-reply-to-names" + "1.11.24") +;; calendar +(define-obsolete-function-alias 'mu4e-icalendar-setup + 'gnus-icalendar-setup '"1.11.22") + +;; mu4e. +(define-obsolete-function-alias 'mu4e-clear-caches #'ignore "1.11.15") + +(provide 'mu4e-obsolete) +;;; mu4e-obsolete.el ends here diff --git a/mu4e/mu4e-org.el b/mu4e/mu4e-org.el new file mode 100644 index 0000000..18d9b66 --- /dev/null +++ b/mu4e/mu4e-org.el @@ -0,0 +1,146 @@ +;;; mu4e-org --- Org-links to mu4e messages/queries -*- lexical-binding: t -*- + +;; Copyright (C) 2012-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Keywords: outlines, hypermedia, calendar, mail + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of 1the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; The expect version here is org 9.x. + +;;; Code: + +(require 'org) +(require 'mu4e-view) +(require 'mu4e-contacts) + + +(defgroup mu4e-org nil + "Settings for the Org mode related functionality in mu4e." + :group 'mu4e + :group 'org) + +(defcustom mu4e-org-link-desc-func + (lambda (msg) (or (plist-get msg :subject) "No subject")) + "Function that takes a msg and returns a description. +This can be used in org capture templates and storing links. + +Example usage: + + (defun my-link-descr (msg) + (let ((subject (or (plist-get msg :subject) + \"No subject\")) + (date (or (format-time-string mu4e-headers-date-format + (mu4e-msg-field msg :date)) + \"No date\"))) + (concat subject \" \" date))) + + (setq mu4e-org-link-desc-func \\='my-link-descr)" + :type '(function) + :group 'mu4e-org) + +(defvar mu4e-org-link-query-in-headers-mode nil + "Prefer linking to the query rather than to the message. +If non-nil, `org-store-link' in `mu4e-headers-mode' links to the +the current query; otherwise, it links to the message at point.") + +;; backward compat until org >= 9.3 is univeral. +(defalias 'mu4e--org-link-store-props + (if (fboundp 'org-link-store-props) + #'org-link-store-props + (with-no-warnings + #'org-store-link-props))) + +(defun mu4e--org-store-link-query () + "Store a link to a mu4e query." + (setq org-store-link-plist nil) ; reset + (mu4e--org-link-store-props + :type "mu4e" + :query (mu4e-last-query) + :date (format-time-string "%FT%T") ;; avoid error + :link (concat "mu4e:query:" (mu4e-last-query)) + :description (format "[%s]" (mu4e-last-query)))) + +(defun mu4e--org-store-link-message (&optional msg) + "Store a link to a mu4e message. +If MSG is non-nil, store a link to MSG, otherwise use `mu4e-message-at-point'." + (setq org-store-link-plist nil) + (let* ((msg (or msg (mu4e-message-at-point))) + (from (car-safe (plist-get msg :from))) + (to (car-safe (plist-get msg :to))) + (date (format-time-string "%FT%T" (plist-get msg :date))) + (msgid (or (plist-get msg :message-id) + (mu4e-error "Cannot link message without message-id"))) + (props `(:type "mu4e" + :date ,date + :from ,(mu4e-contact-full from) + :fromname ,(mu4e-contact-name from) + :fromnameoraddress ,(or (mu4e-contact-name from) + (mu4e-contact-email from)) ;; mu4e-specific + :maildir ,(plist-get msg :maildir) + :message-id ,msgid + :path ,(plist-get msg :path) + :subject ,(plist-get msg :subject) + :to ,(mu4e-contact-full to) + :tonameoraddress ,(or (mu4e-contact-name to) + (mu4e-contact-email to)) ;; mu4e-specific + :link ,(concat "mu4e:msgid:" msgid) + :description ,(funcall mu4e-org-link-desc-func msg)))) + (apply #'mu4e--org-link-store-props props))) + +(defun mu4e-org-store-link () + "Store a link to a mu4e message or query. +It links to the last known query when in `mu4e-headers-mode' with +`mu4e-org-link-query-in-headers-mode' set; otherwise it links to +a specific message, based on its message-id, so that links stay +valid even after moving the message around." + (cond + ((derived-mode-p 'mu4e-view-mode) (mu4e--org-store-link-message)) + ((derived-mode-p 'mu4e-headers-mode) + (if mu4e-org-link-query-in-headers-mode + (mu4e--org-store-link-query) + (mu4e--org-store-link-message))))) + +(defun mu4e-org-open (link) + "Open the org LINK. +Open the mu4e message (for links starting with \"msgid:\") or run +the query (for links starting with \"query:\")." + (require 'mu4e) + (cond + ((string-match "^msgid:\\(.+\\)" link) + (mu4e-view-message-with-message-id (match-string 1 link))) + ((string-match "^query:\\(.+\\)" link) + (mu4e-search (match-string 1 link) current-prefix-arg)) + (t (mu4e-error "Unrecognized link type '%s'" link)))) + +(defun mu4e-org-store-and-capture () + "Store a link to the current message or query. +\(depending on `mu4e-org-link-query-in-headers-mode', and capture +it with org)." + (interactive) + (call-interactively 'org-store-link) + (org-capture)) + +;; install mu4e-link support. +(org-link-set-parameters "mu4e" + :follow #'mu4e-org-open + :store #'mu4e-org-store-link) +(provide 'mu4e-org) +;;; mu4e-org.el ends here diff --git a/mu4e/mu4e-pkg.el.in b/mu4e/mu4e-pkg.el.in new file mode 100644 index 0000000..ed8e733 --- /dev/null +++ b/mu4e/mu4e-pkg.el.in @@ -0,0 +1,7 @@ +;; -*- no-byte-compile: t; -*- +(define-package "mu4e" "@VERSION@" + "part of mu4e, the mu mail user agent" + '((emacs "@EMACS_MIN_VERSION@")) + :authors '(("Dirk-Jan C. Binnema" . "djcb@djcbsoftware.nl")) + :maintainer '("Dirk-Jan C. Binnema" . "djcb@djcbsoftware.nl") + :keywords '("email")) diff --git a/mu4e/mu4e-query-items.el b/mu4e/mu4e-query-items.el new file mode 100644 index 0000000..b2523e7 --- /dev/null +++ b/mu4e/mu4e-query-items.el @@ -0,0 +1,254 @@ +;;; mu4e-query-items.el --- Manage query results -*- lexical-binding: t -*- + +;; Copyright (C) 2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: +;; +;; Managing the last query results / baseline, which we use to get the +;; unread-counts, i.e., query items. `mu4e-query-items` delivers these items, +;; aggregated from various sources. + + +;;; Code: + +;;; Last & baseline query results for bookmarks. +(require 'cl-lib) +(require 'mu4e-helpers) +(require 'mu4e-server) + +(defcustom mu4e-query-rewrite-function 'identity + "Function to rewrite a query. + +It takes a search expression string, and returns a possibly + changed search expression string. + +This function is applied on the search expression just before +searching, and allows users to modify the query. + +For instance, we could change any instance of \"workmail\" into +\"maildir:/long-path-to-work-related-emails\", by setting the function + +\\=(setq mu4e-query-rewrite-function + (lambda(expr) + (replace-regexp-in-string \"workmail\" + \"maildir:/long-path-to-work-related-emails\" expr))) + +It is good to remember that the replacement does not understand +anything about the query, it just does text replacement. + +A word of caution: the function should be deterministic and +always return the same result for a given query (at least within +some \"context\" (see `mu4e-context'). If not, you may get incorrect results +for the various unread counts." + :type 'function + :group 'mu4e-search) + +(defvar mu4e--query-items-baseline nil + "Some previous version of the query-items. +This is used as the baseline to track updates by comparing it to +the latest query-items.") +(defvar mu4e--query-items-baseline-tstamp nil + "Timestamp for when the query-items baseline was updated.") +(defvar mu4e--last-delta-unread 0 "Last notified number.") + +(defun mu4e--bookmark-query (bm) + "Get the query string for some bookmark BM." + (when bm + (let* ((query (or (plist-get bm :query) + (mu4e-warn "No query in %S" bm))) + ;; queries being functions is deprecated, but for now we + ;; still support it. + (query (if (functionp query) (funcall query) query))) + (unless (stringp query) + (mu4e-warn "Could not get query string from %s" bm)) + ;; apparently, non-UTF8 queries exist, i.e., + ;; with maildir names. + (decode-coding-string query 'utf-8 t)))) + +(defun mu4e--query-items-pick-favorite (items) + "Pick the :favorite querty item. +If ITEMS does not yet have a favorite item, pick the first." + (unless (seq-find + (lambda (item) (plist-get item :favorite)) items) + (plist-put (car items) :favorite t)) + items) + +(defvar mu4e--bookmark-items-cached nil "Cached bookmarks query items.") +(defvar mu4e--maildir-items-cached nil "Cached maildirs query items.") + +(declare-function mu4e-bookmarks "mu4e-bookmarks") +(declare-function mu4e-maildir-shortcuts "mu4e-folders") + +(defun mu4e--query-item-display-counts (item) + "Get the count display string for some query-data ITEM." + ;; purely for display, but we need it in the main menu, modeline + ;; so let's keep it consistent. + (cl-destructuring-bind (&key unread hide-unread delta-unread count + &allow-other-keys) item + (if hide-unread + "" + (concat + (propertize (number-to-string unread) + 'face 'mu4e-header-key-face + 'help-echo "Number of unread") + (if (<= delta-unread 0) "" + (propertize (format "(%+d)" delta-unread) 'face + 'mu4e-unread-face)) + "/" + (propertize (number-to-string count) + 'help-echo "Total number"))))) + +(defun mu4e--query-items-refresh (&optional reset-baseline) + "Get the latest query data from the mu4e server. +With RESET-BASELINE, reset the baseline first." + (when reset-baseline + (setq mu4e--query-items-baseline nil + mu4e--query-items-baseline-tstamp nil + mu4e--bookmark-items-cached nil + mu4e--maildir-items-cached nil + mu4e--last-delta-unread 0)) + (mu4e--server-queries + ;; note: we must apply the rewrite function here, since the query does not go + ;; through mu4e-search. + (mapcar (lambda (bm) + (funcall mu4e-query-rewrite-function + (mu4e--bookmark-query bm))) + (seq-filter (lambda (item) + (and (not (or (plist-get item :hide) + (plist-get item :hide-unread))))) + (mu4e-query-items))))) + +(defun mu4e--query-items-queries-handler (_sexp) + "Handler for queries responses from the mu4e-server. +I.e. what we get in response to mu4e--query-items-refresh." + ;; if we cleared the baseline (in mu4e--query-items-refresh) + ;; set it to the latest now. + (unless mu4e--query-items-baseline + (setq mu4e--query-items-baseline (mu4e-server-query-items) + mu4e--query-items-baseline-tstamp (current-time))) + + (setq mu4e--bookmark-items-cached nil + mu4e--maildir-items-cached nil) + (mu4e-query-items) ;; for side-effects + ;; tell the world. + (run-hooks 'mu4e-query-items-updated-hook)) + +;; this makes for O(n*m)... but with typically small(ish) n,m. Perhaps use a +;; hash for last-query-items and baseline-results? +(defun mu4e--query-find-item (query data) + "Find the item in DATA for the given QUERY." + (seq-find (lambda (item) + (equal query (mu4e--bookmark-query item))) + data)) + +(defun mu4e--make-query-items (data type) + "Map the items in DATA to plists with aggregated query information. + +DATA is either the bookmarks or maildirs (user-defined). + +LAST-RESULTS-DATA contains unread/counts we received from the +server, while BASELINE-DATA contains the same but taken at some +earier time. + +The TYPE denotes the category for the query item, a symbol +bookmark or maildir." + (seq-map + (lambda (item) + (let* ((maildir (plist-get item :maildir)) + ;; for maildirs, construct the query + (query (if (equal type 'maildirs) + (format "maildir:\"%s\"" maildir) + (plist-get item :query))) + (query (if (functionp query) (funcall query) query)) + (name (plist-get item :name)) + ;; it is possible that the user has a rewrite function + (effective-query (funcall mu4e-query-rewrite-function query)) + ;; maildir items may have an implicit name + ;; which is the maildir value. + (name (or name (and (equal type 'maildirs) maildir))) + (last-results (mu4e-server-query-items)) + (baseline mu4e--query-items-baseline) + ;; we use the _effective_ query to find the results, + ;; since that's what the server will give to us. + (baseline-item + (mu4e--query-find-item effective-query baseline)) + (last-results-item + (mu4e--query-find-item effective-query last-results)) + (count (or (plist-get last-results-item :count) 0)) + (unread (or (plist-get last-results-item :unread) 0)) + (baseline-count (or (plist-get baseline-item :count) count)) + (baseline-unread (or (plist-get baseline-item :unread) unread)) + (delta-unread (- unread baseline-unread)) + (value + (list + :name name + :query query + :key (plist-get item :key) + :count count + :unread unread + :delta-count (- count baseline-count) + :delta-unread delta-unread))) + ;; remember the *effective* query too; we don't really need it, but + ;; useful for debugging. + (unless (string= query effective-query) + (plist-put value :effective-query effective-query)) + + ;; nil props bring me discomfort + (when (plist-get item :favorite) + (plist-put value :favorite t)) + (when (plist-get item :hide) + (plist-put value :hide t)) + (when (plist-get item :hide-unread) + (plist-put value :hide-unread t)) + value)) + data)) + +(defun mu4e-query-items (&optional type) + "Grab query items of TYPE. + +TYPE is symbol; either bookmarks or maildirs, or nil for both. + +This combines: + - the latest queries data (i.e., `(mu4e-server-query-items)') + - baseline queries data (i.e. `mu4e-baseline') + with the combined queries for `(mu4e-bookmarks)' and + `(mu4e-maildir-shortcuts)' in bookmarks-compatible plists. + +This packages the aggregated information in a format that is convenient +for use in various places." + (cond + ((equal type 'bookmarks) + (or mu4e--bookmark-items-cached + (setq mu4e--bookmark-items-cached + (mu4e--query-items-pick-favorite + (mu4e--make-query-items (mu4e-bookmarks) 'bookmarks))))) + ((equal type 'maildirs) + (or mu4e--maildir-items-cached + (setq mu4e--maildir-items-cached + (mu4e--make-query-items (mu4e-maildir-shortcuts) 'maildirs)))) + ((not type) + (append (mu4e-query-items 'bookmarks) + (mu4e-query-items 'maildirs))) + (t + (mu4e-error "No such type %s" type)))) + +(provide 'mu4e-query-items) +;;; mu4e-query-items.el ends here diff --git a/mu4e/mu4e-search.el b/mu4e/mu4e-search.el new file mode 100644 index 0000000..f42a981 --- /dev/null +++ b/mu4e/mu4e-search.el @@ -0,0 +1,635 @@ +;;; mu4e-search.el --- Search-related functions -*- lexical-binding: t -*- + +;; Copyright (C) 2021,2022 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Search-related functions and a minor-mode. + +;;; Code: + +(require 'seq) +(require 'mu4e-helpers) +(require 'mu4e-message) +(require 'mu4e-bookmarks) +(require 'mu4e-contacts) +(require 'mu4e-lists) +(require 'mu4e-mark) +(require 'mu4e-query-items) + + +;;; Configuration +(defgroup mu4e-search nil + "Search-related settings." + :group 'mu4e) + +(defcustom mu4e-search-results-limit 500 + "Maximum number of results to show. +This affects performance, especially when +`mu4e-summary-include-related' is non-nil. +Set to -1 for no limits." + :type '(choice (const :tag "Unlimited" -1) + (integer :tag "Limit")) + :group 'mu4e-search) + +(defcustom mu4e-search-full nil + "Whether to search for all results. +If this is nil, search for up to `mu4e-search-results-limit')" + :type 'boolean + :group 'mu4e-search) + +(defcustom mu4e-search-threads t + "Whether to calculate threads for the search results." + :type 'boolean + :group 'mu4e-search) + +(defcustom mu4e-search-include-related t + "Whether to include \"related\" messages in queries. +With this option set to non-nil, not just return the matches for +a searches, but also messages that are related (through their +references) to these messages. This can be useful e.g. to include +sent messages into message threads." + :type 'boolean + :group 'mu4e-search) + +(defcustom mu4e-search-skip-duplicates t + "Whether to skip duplicate messages. +With this option set to non-nil, show only one of duplicate +messages. This is useful when you have multiple copies of the same +message, which is a common occurrence for example when using Gmail +and offlineimap." + :type 'boolean + :group 'mu4e-search) + +(defvar mu4e-search-hide-predicate nil + "Predicate function to hide matching headers. +Either nil or a function taking one message plist parameter and +which which return non-nil for messages that should be hidden from +the search results. Also see `mu4e-search-hide-enabled'. + +Example that hides all trashed messages: + + (setq mu4e-search-hide-predicate + (lambda (msg) + (member \\='trashed (mu4e-message-field msg :flags)))).") + +(defvar mu4e-search-hide-enabled t + "Whether `mu4e-search-hide-predicate' should be active. +This can be used to toggle use of the predicate through + `mu4e-search-toggle-property'.") + + +(defcustom mu4e-search-sort-field :date + "Field to sort the headers by. A symbol: +one of: `:date', `:subject', `:size', `:prio', `:from', `:to.', +`:list'. + +Note that when threading is enabled (through +`mu4e-search-threads'), the headers are exclusively sorted +chronologically (`:date') by the newest message in the thread." + :type '(radio (const :date) + (const :subject) + (const :size) + (const :prio) + (const :from) + (const :to) + (const :list)) + :group 'mu4e-search) + +(defcustom mu4e-search-sort-direction 'descending + "Direction to sort by; a symbol either `descending' (sorting + Z->A) or `ascending' (sorting A->Z)." + :type '(radio (const ascending) + (const descending)) + :group 'mu4e-search) + +;; mu4e-query-rewrite-function lives in mu4e-query-items.el +;; to avoid circular deps. + +(defcustom mu4e-search-bookmark-hook nil + "Hook run just after invoking a bookmarked search. + +This function receives the query as its parameter, before any +rewriting as per `mu4e-query-rewrite-function' has taken place. + +The reason to use this instead of `mu4e-search-hook' is +if you only want to execute a hook when a search is entered via a +bookmark, e.g. if you'd like to treat the bookmarks as a custom +folder and change the options for the search." + :type 'hook + :group 'mu4e-search) + +(defcustom mu4e-search-hook nil + "Hook run just before executing a new search operation. +This function receives the query as its parameter, before any +rewriting as per `mu4e-query-rewrite-function' has taken place + +This is a more general hook facility than the +`mu4e-search-bookmark-hook'. It gets called on every +executed search, not just those that are invoked via bookmarks, +but also manually invoked searches." + :type 'hook + :group 'mu4e-search) + +;; Internals + +;;; History +(defvar mu4e--search-query-past nil + "Stack of queries before the present one.") +(defvar mu4e--search-query-future nil "Stack of queries after the present one.") +(defvar mu4e--search-query-stack-size 20 + "Maximum size for the query stacks.") +(defvar mu4e--search-last-query nil + "The present (most recent) query.") + + + +;;; Interactive functions +(declare-function mu4e--search-execute "mu4e-headers") + +(defvar mu4e--search-view-target nil + "Whether to automatically view (open) the target message.") +(defvar mu4e--search-msgid-target nil + "Message-id to jump to after the search has finished.") + + +(defun mu4e-search (&optional expr prompt edit ignore-history msgid show) + "Search for query EXPR. + +Switch to the output buffer for the results. This is an +interactive function which ask user for EXPR. PROMPT, if non-nil, +is the prompt used by this function (default is \"Search for:\"). +If EDIT is non-nil, instead of executing the query for EXPR, let +the user edit the query before executing it. + +If IGNORE-HISTORY is true, do *not* update the query history +stack. If MSGID is non-nil, attempt to move point to the first +message with that message-id after searching. If SHOW is non-nil, +show the message with MSGID." + (interactive) + (let* ((prompt (mu4e-format (or prompt "Search for: "))) + (expr + (if (or (null expr) edit) + (mu4e-search-read-query prompt expr) + expr))) + (mu4e-mark-handle-when-leaving) + (mu4e--search-execute expr ignore-history) + (setq mu4e--search-msgid-target msgid + mu4e--search-view-target show) + (mu4e--modeline-update))) + +(defun mu4e-search-edit () + "Edit the last search expression." + (interactive) + (mu4e-search mu4e--search-last-query nil t)) + +(defun mu4e-search-bookmark (&optional expr edit) + "Search using some bookmarked query EXPR. +If EDIT is non-nil, let the user edit the bookmark before starting +the search." + (interactive) + (let* ((expr + (or expr + (mu4e-ask-bookmark + (if edit "Select bookmark: " "Bookmark: ")))) + (expr (if (functionp expr) (funcall expr) expr)) + (fav (mu4e--bookmark-query (mu4e-bookmark-favorite)))) + ;; reset baseline when searching for the favorite bookmark query + (when (and fav (string= fav expr)) + (mu4e--query-items-refresh 'reset-baseline)) + + (run-hook-with-args 'mu4e-search-bookmark-hook expr) + (mu4e-search expr (when edit "Edit query: ") edit))) + +(defun mu4e-search-bookmark-edit () + "Edit an existing bookmark before executing it." + (interactive) + (mu4e-search-bookmark nil t)) + + +(defun mu4e-search-maildir (maildir &optional edit) + "Search the messages in MAILDIR. +The user is prompted to ask what maildir. If prefix-argument EDIT +is given, offer to edit the search query before executing it." + (interactive + (let ((maildir (mu4e-ask-maildir "Jump to maildir: "))) + (list maildir current-prefix-arg))) + (when maildir + (let* ((query (format "maildir:\"%s\"" maildir)) + (query (if edit + (mu4e-search-read-query "Refine query: " query) query))) + (mu4e-mark-handle-when-leaving) + (mu4e-search query)))) + +(defun mu4e-search-narrow(&optional filter) + "Narrow the last search. +Do so by appending search expression FILTER to the last search +expression. Note that you can go back to the previous +query (effectively, \"widen\" it), with `mu4e-search-prev'." + (interactive + (let ((filter + (read-string (mu4e-format "Narrow down to: ") + nil 'mu4e~headers-search-hist nil t))) + (list filter))) + (unless mu4e--search-last-query + (mu4e-warn "There's nothing to filter")) + (mu4e-search (format "(%s) AND (%s)" mu4e--search-last-query filter))) + +(defun mu4e--search-push-query (query where) + "Push QUERY to one of the query stacks. +WHERE is a symbol telling us where to push; it's a symbol, either +`future' or `past'. Also removes duplicates and truncates to +limit the stack size." + (let ((stack + (pcase where + ('past mu4e--search-query-past) + ('future mu4e--search-query-future)))) + ;; only add if not the same item + (unless (and stack (string= (car stack) query)) + (push query stack) + ;; limit the stack to `mu4e--search-query-stack-size' elements + (when (> (length stack) mu4e--search-query-stack-size) + (setq stack (cl-subseq stack 0 mu4e--search-query-stack-size))) + ;; remove all duplicates of the new element + (seq-remove (lambda (elm) (string= elm (car stack))) (cdr stack)) + ;; update the stacks + (pcase where + ('past (setq mu4e--search-query-past stack)) + ('future (setq mu4e--search-query-future stack)))))) + +(defun mu4e--search-pop-query (whence) + "Pop a query from the stack. +WHENCE is a symbol telling us where to get it from, either `future' +or `past'." + (pcase whence + ('past + (unless mu4e--search-query-past + (mu4e-warn "No more previous queries")) + (pop mu4e--search-query-past)) + ('future + (unless mu4e--search-query-future + (mu4e-warn "No more next queries")) + (pop mu4e--search-query-future)))) + +(defun mu4e-search-rerun () + "Re-run the search for the last search expression." + (interactive) + ;; if possible, try to return to the same message + (let* ((msg (mu4e-message-at-point t)) + (msgid (and msg (mu4e-message-field msg :message-id)))) + (mu4e-search mu4e--search-last-query nil nil t msgid))) + +(defun mu4e--search-query-navigate (whence) + "Execute the previous query from the query stacks. +WHENCE determines where the query is taken from and is a symbol, +either `future' or `past'." + (let ((query (mu4e--search-pop-query whence)) + (where (if (eq whence 'future) 'past 'future))) + (when query + (mu4e--search-push-query mu4e--search-last-query where) + (mu4e-search query nil nil t)))) + +(defun mu4e-search-next () + "Execute the next query from the query stack." + (interactive) + (mu4e--search-query-navigate 'future)) + +(defun mu4e-search-prev () + "Execute the previous query from the query stacks." + (interactive) + (mu4e--search-query-navigate 'past)) + +;; forget the past so we don't repeat it :/ +(defun mu4e-search-forget () + "Forget the search history." + (interactive) + (setq mu4e--search-query-past nil + mu4e--search-query-future nil) + (mu4e-message "Query history cleared")) + +(defun mu4e-last-query () + "Get the most recent query or nil if there is none." + mu4e--search-last-query) + +;;; Completion for queries + +(defvar mu4e--search-hist nil "History list of searches.") +(defvar mu4e-minibuffer-search-query-map + (let ((map (copy-keymap minibuffer-local-map))) + (define-key map (kbd "TAB") #'completion-at-point) + map) + "The keymap for reading a search query.") + +(defun mu4e-search-read-query (prompt &optional initial-input) + "Read a query with completion using PROMPT and INITIAL-INPUT." + (minibuffer-with-setup-hook + (lambda () + (setq-local completion-at-point-functions + #'mu4e--search-query-completion-at-point) + (use-local-map mu4e-minibuffer-search-query-map)) + (read-string prompt initial-input 'mu4e--search-hist))) + +(defconst mu4e--search-query-keywords + '("and" "or" "not" + "from:" "to:" "cc:" "bcc:" "contact:" "recip:" "date:" "subject:" "body:" + "list:" "maildir:" "flag:" "mime:" "file:" "prio:" "tag:" "msgid:" + "size:" "embed:")) + +(defun mu4e--search-completion-contacts-action (match _status) + "Delete contact alias from contact autocompletion, leaving just email address. +Implements the `completion-extra-properties' :exit-function' which +requires a function with arguments string MATCH and completion +status, STATUS." + (let ((contact-email (replace-regexp-in-string "^.*<\\|>$" "" match))) + (delete-char (- (length match))) + (insert contact-email))) + +(defun mu4e--search-query-completion-at-point () + "Provide completion when entering search expressions." + (cond + ((not (looking-back "[:\"][^ \t]*" nil)) + (let ((bounds (bounds-of-thing-at-point 'word))) + (list (or (car bounds) (point)) + (or (cdr bounds) (point)) + mu4e--search-query-keywords))) + ((looking-back "flag:\\(\\w*\\)" nil) + (list (match-beginning 1) + (match-end 1) + '("attach" "draft" "flagged" "list" "new" "passed" "replied" + "seen" "trashed" "unread" "encrypted" "signed" "personal"))) + ((looking-back "maildir:\\([a-zA-Z0-9/.]*\\)" nil) + (list (match-beginning 1) + (match-end 1) + (mapcar (lambda (dir) + ;; Quote maildirs with whitespace in their name, e.g., + ;; maildir:"Foobar/Junk Mail". + (if (string-match-p "[[:space:]]" dir) + (concat "\"" dir "\"") + dir)) + (mu4e-get-maildirs)))) + ((looking-back "prio:\\(\\w*\\)" nil) + (list (match-beginning 1) + (match-end 1) + (list "high" "normal" "low"))) + ((looking-back "mime:\\([a-zA-Z0-9/-]*\\)" nil) + (list (match-beginning 1) + (match-end 1) + (when (fboundp 'mailcap-mime-types) (mailcap-mime-types)))) + ((looking-back "\\(from\\|to\\|cc\\|bcc\\|contact\\|recip\\):\\([a-zA-Z0-9/.@]*\\)" nil) + (list (match-beginning 2) + (match-end 2) + mu4e--contacts-set + :exit-function + #'mu4e--search-completion-contacts-action)) + ((looking-back "list:\\([a-zA-Z0-9/.@]*\\)" nil) + (list (match-beginning 1) + (match-end 1) + mu4e--lists-hash)))) + +;;; Interactive functions +(defun mu4e-search-change-sorting (&optional field dir) + "Change the sorting/threading parameters. +FIELD is the field to sort by; DIR is a symbol: either +`ascending', `descending', t (meaning: if FIELD is the same as +the current sortfield, change the sort-order) or nil (ask the +user). + +When threads are enabled (`mu4e-search-threads'), you can only sort +by the `:date' field." + (interactive) + (let* ((choices ;; with threads enabled, you can only sort by *date* + (if mu4e-search-threads + '(("date" . :date)) + '(("date" . :date) + ("from" . :from) + ("list" . :list) + ("maildir" . :maildir) + ("prio" . :prio) + ("zsize" . :size) + ("subject" . :subject) + ("to" . :to)))) + (field + (or field + (mu4e-read-option "Sortfield: " choices))) + ;; note: 'sortable' is either a boolean (meaning: if non-nil, this is + ;; sortable field), _or_ another field (meaning: sort by this other + ;; field). + (sortable (plist-get (cdr (assoc field mu4e-header-info)) :sortable)) + ;; error check + (sortable + (if sortable + sortable + (mu4e-error "Not a sortable field"))) + (sortfield (if (booleanp sortable) field sortable)) + (dir + (cl-case dir + ((ascending descending) dir) + ;; change the sort order if field = curfield + (t + (if (eq sortfield mu4e-search-sort-field) + (if (eq mu4e-search-sort-direction 'ascending) + 'descending 'ascending) + 'descending))))) + (setq + mu4e-search-sort-field sortfield + mu4e-search-sort-direction dir) + (mu4e-message "Sorting by %s (%s)" + (symbol-name sortfield) + (symbol-name mu4e-search-sort-direction)) + (mu4e-search-rerun))) + +(defun mu4e-search-toggle-property (&optional dont-refresh) + "Toggle some aspect of search. +When prefix-argument DONT-REFRESH is non-nil, do not refresh the +last search with the new setting." + (interactive "P") + (let* ((toggles '(("fFull-search" . mu4e-search-full) + ("rInclude-related" . mu4e-headers-include-related) + ("tShow threads" . mu4e-search-threads) + ("uSkip duplicates" . mu4e-search-skip-duplicates) + ("pHide-predicate" . mu4e-search-hide-enabled))) + (toggles (seq-map + (lambda (cell) + (cons + (concat (car cell) + (format" (%s)" + (if (symbol-value (cdr cell)) "on" "off"))) + (cdr cell))) + toggles)) + (choice (mu4e-read-option "Toggle property " toggles))) + (when choice + (set choice (not (symbol-value choice))) + (mu4e-message "Set `%s' to %s" (symbol-name choice) (symbol-value choice)) + (mu4e--modeline-update) + (unless dont-refresh + (mu4e-search-rerun))))) + +(defvar mu4e-search-threaded-label '("T" . "Ⓣ") + "Non-fancy and fancy labels to indicate threaded search in the mode-line.") +(defvar mu4e-search-full-label '("F" . "Ⓕ") + "Non-fancy and fancy labels to indicate full search in the mode-line.") +(defvar mu4e-search-related-label '("R" . "Ⓡ") + "Non-fancy and fancy labels to indicate related search in the mode-line.") +(defvar mu4e-search-skip-duplicates-label '("U" . "Ⓤ") ;; 'U' for 'unique' + "Non-fancy and fancy labels for include-related search in the mode-line.") +(defvar mu4e-search-hide-label '("H" . "Ⓗ") + "Non-fancy and fancy labels to indicate header-hiding is active in +the mode-line.") + +(defun mu4e--search-modeline-item () + "Get mu4e-search modeline item." + (let* ((label (lambda (label-cons) + (if mu4e-use-fancy-chars + (cdr label-cons) (car label-cons)))) + (props + `((,mu4e-search-full ,mu4e-search-full-label + "Full search") + (,mu4e-search-include-related + ,mu4e-search-related-label + "Include related messages") + (,mu4e-search-threads + ,mu4e-search-threaded-label + "Show message threads") + (,mu4e-search-skip-duplicates + ,mu4e-search-skip-duplicates-label + "Skip duplicate messages") + (,mu4e-search-hide-enabled + ,mu4e-search-hide-label + "Enable message hide predicate"))) + ;; can we fin find a bookmark corresponding + ;; with this query? + (bookmark + (and mu4e-modeline-prefer-bookmark-name + (seq-find (lambda (item) + (string= + mu4e--search-last-query + (or (plist-get item :effective-query) + (plist-get item :query)))) + (mu4e-query-items 'bookmarks))))) + (concat + (propertize + (mapconcat + (lambda (cell) + (when (nth 0 cell) (funcall label (nth 1 cell)))) + props "") + 'help-echo (concat "mu4e search properties legend\n\n" + (mapconcat + (lambda (cell) + (format "%s %s (%s)" + (funcall label (nth 1 cell)) + (nth 2 cell) + (if (nth 0 cell) "yes" : "no"))) + props "\n"))) + " [" + (propertize + (if bookmark ;; show the bookmark name instead of the query? + (plist-get bookmark :name) + mu4e--search-last-query) + 'face 'mu4e-title-face + 'help-echo (format "mu4e query:\n\t%s" mu4e--search-last-query)) + "]"))) + +(defun mu4e-search-query (&optional edit) + "Select a search query through `completing-read'. + +If prefix-argument EDIT is non-nil, allow for editing the chosen +query before submitting it." + (interactive "P") + (let* ((candidates (seq-map (lambda (item) + (cons (plist-get item :name) item)) + (mu4e-query-items))) + (longest-name + (seq-max (seq-map (lambda (c) (length (car c))) candidates))) + (longest-query + (seq-max (seq-map (lambda (c) (length (plist-get (cdr c) :query))) + candidates))) + + (annotation-func + (lambda (candidate) + (let* ((item (cdr-safe (assoc candidate candidates))) + (name (propertize (or (plist-get item :name) "") + 'face 'mu4e-header-key-face)) + (query (propertize (or (plist-get item :query) "") + 'face 'mu4e-header-value-face))) + (concat + " " + (make-string (- longest-name (length name)) ?\s) + query + (make-string (- longest-query (length query)) ?\s) + " " + (mu4e--query-item-display-counts item))))) + (completion-extra-properties + `(:annotation-function ,annotation-func)) + (chosen (completing-read "Query: " candidates)) + (query (or (plist-get (cdr-safe (assoc chosen candidates)) :query) + (mu4e-warn "No query for %s" chosen)))) + (mu4e-search-bookmark query edit))) + +(defvar mu4e-search-minor-mode-map + (let ((map (make-sparse-keymap))) + (define-key map "s" #'mu4e-search) + (define-key map "S" #'mu4e-search-edit) + (define-key map "/" #'mu4e-search-narrow) + + (define-key map (kbd "<M-left>") #'mu4e-search-prev) + (define-key map (kbd "\\") #'mu4e-search-prev) + (define-key map (kbd "<M-right>") #'mu4e-search-next) + + (define-key map "O" #'mu4e-search-change-sorting) + (define-key map "P" #'mu4e-search-toggle-property) + + (define-key map "b" #'mu4e-search-bookmark) + (define-key map "B" #'mu4e-search-bookmark-edit) + + (define-key map "c" #'mu4e-search-query) + + (define-key map "j" #'mu4e-search-maildir) + map) + "Keymap for mu4e-search-minor-mode.") + +(define-minor-mode mu4e-search-minor-mode + "Mode for searching for messages." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap mu4e-search-minor-mode-map) + +(defvar mu4e--search-menu-items + '("--" + ["Search" mu4e-search + :help "Search using expression"] + ["Search bookmark" mu4e-search-bookmark + :help "Show messages matching some bookmark query"] + ["Search maildir" mu4e-search-maildir + :help "Show messages in some maildir"] + ["Choose query" mu4e-search-query + :help "Show messages for some query"] + ["Previous query" mu4e-search-prev + :help "Run previous query"] + ["Next query" mu4e-search-next + :help "Run next query"] + ["Narrow search" mu4e-search-narrow + :help "Narrow the search query"]) + "Easy menu items for search.") + +(provide 'mu4e-search) +;;; mu4e-search.el ends here diff --git a/mu4e/mu4e-server.el b/mu4e/mu4e-server.el new file mode 100644 index 0000000..a280f09 --- /dev/null +++ b/mu4e/mu4e-server.el @@ -0,0 +1,712 @@ +;;; mu4e-server.el --- Control mu server from mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: + +(require 'mu4e-helpers) + + +;;; Configuration +(defcustom mu4e-mu-home nil + "Location of an alternate mu home dir. +If not set, use the defaults, based on the XDG Base Directory +Specification. + +Changes to this value only take effect after (re)starting the mu +session." + :group 'mu4e + :type '(choice (const :tag "Default location" nil) + (directory :tag "Specify location")) + :safe 'stringp) + +(defcustom mu4e-mu-binary (executable-find "mu") + "Path to the mu-binary to use. + +Changes to this value only take effect after (re)starting the mu +session." + :type '(file :must-match t) + :group 'mu4e + :safe 'stringp) + +(defcustom mu4e-mu-debug nil + "Whether to run the mu binary in debug-mode. +Setting this to t increases the amount of information in the log. + +Changes to this value only take effect after (re)starting the mu +session." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-change-filenames-when-moving nil + "Change message file names when moving them. + +When moving messages to different folders, normally mu/mu4e keep +the base filename the same (the flags-part of the filename may +change still). With this option set to non-nil, mu4e instead +changes the filename. + +This latter behavior works better with some +IMAP-synchronization programs such as mbsync; the default works +better with e.g. offlineimap." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-mu-allow-temp-file nil + "Allow using temp-files for optimizing mu <-> mu4e communication. + +Some commands - in particular \"find\" and \"contacts\" - return +big s-expressions; and it turns out that reading those is faster +by passing them through a temp file rather than through normal +stdin/stdout channel - esp. on the (common case) where the +file-system for temp-files is in-memory. + +To see if the helps, you can benchmark the rendering with + (setq mu4e-headers-report-render-time t) + +and compare the results with `mu4e-mu-allow-temp' set and unset. + +Note: for a change to this variable to take effect, you need to +stop/start mu4e." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + + +;; Cached data +(defvar mu4e-maildir-list) + + +;; Handlers are not strictly internal, but are not meant +;; for overriding outside mu4e. The are mainly for breaking +;; dependency cycles. + +(defvar mu4e-error-func nil + "Function called for each error received. +The function is passed an error plist as argument. See +`mu4e--server-filter' for the format.") + +(defvar mu4e-update-func nil + "Function called for each :update sexp returned. +The function is passed a msg sexp as argument. +See `mu4e--server-filter' for the format.") + +(defvar mu4e-remove-func nil + "Function called for each :remove sexp returned. +This happens when some message has been deleted. The function is +passed the docid of the removed message.") + +(defvar mu4e-sent-func nil + "Function called for each :sent sexp received. +This happens when some message has been sent. The function is +passed the docid and the draft-path of the sent message.") + +(defvar mu4e-view-func nil + "Function called for each single-message sexp. +The function is passed a message sexp as argument. See +`mu4e--server-filter' for the format.") + +(defvar mu4e-headers-append-func nil + "Function called with a list of headers to append. +The function is passed a list of message plists as argument. See +See `mu4e--server-filter' for the details.") + +(defvar mu4e-found-func nil + "Function called for when we received a :found sexp. +This happens after the headers have been returned, to report on +the number of matches. See `mu4e--server-filter' for the format.") + +(defvar mu4e-erase-func nil + "Function called we receive an :erase sexp. +This before new headers are displayed, to clear the current +headers buffer. See `mu4e--server-filter' for the format.") + +(defvar mu4e-info-func nil + "Function called for each (:info type ....) sexp received. +from the server process.") + +(defvar mu4e-pong-func nil + "Function called for each (:pong type ....) sexp received.") + +(defvar mu4e-queries-func nil + "Function called for each (:queries type ....) sexp received.") + +(defvar mu4e-contacts-func nil + "A function called for each (:contacts (<list-of-contacts>)) +sexp received from the server process.") + + +;;; Dealing with Server properties +(defvar mu4e--server-props nil + "Metadata we receive from the mu4e server.") + +(defun mu4e-server-properties () + "Get the server metadata plist." + mu4e--server-props) + +(defun mu4e-root-maildir() + "Get the root maildir." + (or (and mu4e--server-props + (plist-get mu4e--server-props :root-maildir)) + (mu4e-error "Root maildir unknown; did you start mu4e?"))) + +(defun mu4e-database-path() + "Get the root maildir." + (or (and mu4e--server-props + (plist-get mu4e--server-props :database-path)) + (mu4e-error "Root maildir unknown; did you start mu4e?"))) + +(defun mu4e-server-version() + "Get the root maildir." + (or (and mu4e--server-props + (plist-get mu4e--server-props :version)) + (mu4e-error "Version unknown; did you start mu4e?"))) + +;;; remember queries result. +(defvar mu4e--server-query-items nil + "Query items results we receive from the mu4e server. +Those are the results from the counting-queries +for bookmarks and maildirs.") + +(defun mu4e-server-query-items () + "Get the latest server query items." + mu4e--server-query-items) + + +;;; Handling raw server data + +(defvar mu4e--server-buf nil + "Buffer (string) for data received from the backend.") +(defconst mu4e--server-name " *mu4e-server*" + "Name of the server process, buffer.") +(defvar mu4e--server-process nil + "The mu-server process.") + +;; dealing with the length cookie that precedes expressions +(defconst mu4e--server-cookie-pre "\376" + "Each expression starts with a length cookie: +<`mu4e--server-cookie-pre'><length-in-hex><`mu4e--server-cookie-post'>.") +(defconst mu4e--server-cookie-post "\377" + "Each expression starts with a length cookie: +<`mu4e--server-cookie-pre'><length-in-hex><`mu4e--server-cookie-post'>.") +(defconst mu4e--server-cookie-matcher-rx + (concat mu4e--server-cookie-pre "\\([[:xdigit:]]+\\)" + mu4e--server-cookie-post) + "Regular expression matching the length cookie. +Match 1 will be the length (in hex).") + +(defun mu4e-running-p () + "Whether mu4e is running. +Checks whether the server process is live." + (and mu4e--server-process + (memq (process-status mu4e--server-process) + '(run open listen connect stop)) t)) + +(defsubst mu4e--server-eat-sexp-from-buf () + "Eat the next s-expression from `mu4e--server-buf'. +Note: this is a string, not an emacs-buffer. `mu4e--server-buf gets +its contents from the mu-servers in the following form: + <`mu4e--server-cookie-pre'><length-in-hex><`mu4e--server-cookie-post'> +Function returns this sexp, or nil if there was none. +`mu4e--server-buf' is updated as well, with all processed sexp data +removed." + (ignore-errors ;; the server may die in the middle... + (let ((b (string-match mu4e--server-cookie-matcher-rx mu4e--server-buf)) + (sexp-len) (objcons)) + (when b + (setq sexp-len (string-to-number (match-string 1 mu4e--server-buf) 16)) + ;; does mu4e--server-buf contain the full sexp? + (when (>= (length mu4e--server-buf) (+ sexp-len (match-end 0))) + ;; clear-up start + (setq mu4e--server-buf (substring mu4e--server-buf (match-end 0))) + ;; note: we read the input in binary mode -- here, we take the part + ;; that is the sexp, and convert that to utf-8, before we interpret + ;; it. + (setq objcons (read-from-string + (decode-coding-string + (substring mu4e--server-buf 0 sexp-len) + 'utf-8 t))) + (when objcons + (setq mu4e--server-buf (substring mu4e--server-buf sexp-len)) + (car objcons))))))) + +(defun mu4e--server-plist-get (plist key) + "Like `plist-get' but load data from file if it is a string. + +I.e. (mu4e--server-plist-get (:foo bar) :foo) + => bar +but + (mu4e--server-plist-get (:foo \"/tmp/data.eld\") :foo) + => evaluates the contents of /tmp/data.eld + (and deletes the file afterward). + +This for the few sexps we get from the mu server that support this +(headers, contacts, maildirs)." + ;; XXX: perhaps re-use the same buffer? + (let ((val (plist-get plist key))) + (if (stringp val) + (with-temp-buffer + (insert-file-contents val) + (goto-char (point-min)) + (delete-file val) + (read (current-buffer))) + val))) + + +(defun mu4e--server-filter (_proc str) + "Filter string STR from PROC. +This processes the \"mu server\" output. It accumulates the +strings into valid sexpsv and evaluating those. + +The server output is as follows: + + 1. an error + (:error 2 :message \"unknown command\") + ;; eox + => passed to `mu4e-error-func'. + + 2a. a header exp looks something like: + (:headers + ( ;; message 1 + :docid 1585 + :from ((\"Donald Duck\" . \"donald@example.com\")) + :to ((\"Mickey Mouse\" . \"mickey@example.com\")) + :subject \"Wicked stuff\" + :date (20023 26572 0) + :size 15165 + :references (\"200208121222.g7CCMdb80690@msg.id\") + :in-reply-to \"200208121222.g7CCMdb80690@msg.id\" + :message-id \"foobar32423847ef23@pluto.net\" + :maildir: \"/archive\" + :path \"/home/mickey/Maildir/inbox/cur/1312_3.32282.pluto,4cd5bd4e9:2,\" + :priority high + :flags (new unread) + :meta <meta-data> + ) + ( .... more messages ) +) +;; eox + => this will be passed to `mu4e-headers-append-func'. + + 2b. After the list of headers has been returned (see 2a.), + we'll receive a sexp that looks like + (:found <n>) with n the number of messages found. The <n> will be + passed to `mu4e-found-func'. + + 3. a view looks like: + (:view <msg-sexp>) + => the <msg-sexp> (see 2.) will be passed to `mu4e-view-func'. + the <msg-sexp> also contains :body-txt and/or :body-html + + 4. a database update looks like: + (:update <msg-sexp> :move <nil-or-t>) + like :header + + => the <msg-sexp> (see 2.) will be passed to + `mu4e-update-func', :move tells us whether this is a move to + another maildir, or merely a flag change. + + 5. a remove looks like: + (:remove <docid>) + => the docid will be passed to `mu4e-remove-func' + + 6. a compose looks like: + (:compose <reply|forward|edit|new> [:original<msg-sexp>] [:include <attach>]) + `mu4e-compose-func'. :original looks like :view." + (mu4e-log 'misc "* Received %d byte(s)" (length str)) + (setq mu4e--server-buf (concat mu4e--server-buf str)) ;; update our buffer + (let ((sexp (mu4e--server-eat-sexp-from-buf))) + (with-local-quit + (while sexp + (mu4e-log 'from-server "%s" sexp) + (cond + ;; a list of messages (after a find command) + ((plist-get sexp :headers) + (funcall mu4e-headers-append-func + (mu4e--server-plist-get sexp :headers))) + + ;; the found sexp, we receive after getting all the headers + ((plist-get sexp :found) + (funcall mu4e-found-func (plist-get sexp :found))) + + ;; viewing a specific message + ((plist-get sexp :view) + (funcall mu4e-view-func (plist-get sexp :view))) + + ;; receive an erase message + ((plist-get sexp :erase) + (funcall mu4e-erase-func)) + + ;; receive a :sent message + ((plist-get sexp :sent) + (funcall mu4e-sent-func + (plist-get sexp :docid) + (plist-get sexp :path))) + + ;; received a pong message + ((plist-get sexp :pong) + (setq mu4e--server-props (plist-get sexp :props)) + (funcall mu4e-pong-func sexp)) + + ;; receive queries info + ((plist-get sexp :queries) + (setq mu4e--server-query-items (plist-get sexp :queries)) + (funcall mu4e-queries-func sexp)) + + ;; received a contacts message + ;; note: we use 'member', to match (:contacts nil) + ((plist-member sexp :contacts) + (funcall mu4e-contacts-func + (mu4e--server-plist-get sexp :contacts) + (plist-get sexp :tstamp))) + + ;; something got moved/flags changed + ((plist-get sexp :update) + (funcall mu4e-update-func + (plist-get sexp :update) + (plist-get sexp :move) + (plist-get sexp :maybe-view))) + + ;; a message got removed + ((plist-get sexp :remove) + (funcall mu4e-remove-func (plist-get sexp :remove))) + + ;; get some info + ((plist-get sexp :info) + (funcall mu4e-info-func sexp)) + + ;; get some data + ((plist-get sexp :maildirs) + (setq mu4e-maildir-list (mu4e--server-plist-get sexp :maildirs))) + + ;; receive an error + ((plist-get sexp :error) + (funcall mu4e-error-func + (plist-get sexp :error) + (plist-get sexp :message))) + + (t (mu4e-message "Unexpected data from server [%S]" sexp))) + + (setq sexp (mu4e--server-eat-sexp-from-buf)))))) + +(defun mu4e--kill-stale () + "Kill stale mu4e server process. +As per issue #2198." + (seq-each + (lambda(proc) + (when (and (process-live-p proc) + (string-prefix-p mu4e--server-name (process-name proc))) + (mu4e-message "killing stale mu4e server") + (ignore-errors + (signal-process proc 'SIGINT) ;; nicely + (sit-for 1.0) + (signal-process proc 'SIGKILL)))) ;; forcefully + (process-list))) + +(defun mu4e--server-args() + "Return the command line args for the command to start the mu4e-server." + ;; [--debug] server [--muhome=..] + (seq-filter #'identity ;; filter out nil + `(,(when mu4e-mu-debug "--debug") + "server" + ,(when mu4e-mu-allow-temp-file "--allow-temp-file") + ,(when mu4e-mu-home (format "--muhome=%s" mu4e-mu-home))))) + +(defun mu4e--version-check () + ;; sanity-check 1 + (let ((default-directory temporary-file-directory)) ;;ensure it's local. + (unless (and mu4e-mu-binary (file-executable-p mu4e-mu-binary)) + (mu4e-error + "Cannot find mu, please set `mu4e-mu-binary' to the mu executable path")) + ;; sanity-check 2 + (let ((version (let ((s (shell-command-to-string + (concat mu4e-mu-binary " --version")))) + (and (string-match "version \\([.0-9]+\\)" s) + (match-string 1 s))))) + (if (not (string= version mu4e-mu-version)) + (mu4e-error + (concat + "Found mu version %s, but mu4e needs version %s" + "; please set `mu4e-mu-binary' " + "accordingly") + version mu4e-mu-version) + (mu4e-message "Found mu version %s" version))))) + +(defun mu4e-server-repl () + "Start a mu4e-server repl. + +This is meant for debugging/testing - the repl is designed for +machines, not for humans. + +You cannot run the repl when mu4e is running (or vice-versa)." + (interactive) + (if (mu4e-running-p) + (mu4e-error "Cannot run repl when mu4e is running") + (progn + (mu4e--version-check) + (let ((cmd (string-join (cons mu4e-mu-binary (mu4e--server-args)) " "))) + (term cmd) + (rename-buffer "*mu4e-repl*" 'unique) + (message "invoked: '%s'" cmd))))) + +(defun mu4e--server-start () + "Start the mu server process." + (mu4e--version-check) + ;; kill old/stale servers, if any. + (mu4e--kill-stale) + (let* ((process-connection-type nil) ;; use a pipe + (args (mu4e--server-args))) + (setq mu4e--server-buf "") + (mu4e-log 'misc "* invoking '%s' with parameters %s" mu4e-mu-binary + (mapconcat (lambda (arg) (format "'%s'" arg)) args " ")) + (setq mu4e--server-process (apply 'start-process + mu4e--server-name mu4e--server-name + mu4e-mu-binary args)) + ;; register a function for (:info ...) sexps + (unless mu4e--server-process + (mu4e-error "Failed to start the mu4e backend")) + (set-process-query-on-exit-flag mu4e--server-process nil) + (set-process-coding-system mu4e--server-process 'binary 'utf-8-unix) + (set-process-filter mu4e--server-process 'mu4e--server-filter) + (set-process-sentinel mu4e--server-process 'mu4e--server-sentinel))) + +(defun mu4e--server-kill () + "Kill the mu server process." + (let* ((buf (get-buffer mu4e--server-name)) + (proc (and (buffer-live-p buf) (get-buffer-process buf)))) + (when proc + (mu4e-message "shutting down") + (set-process-filter mu4e--server-process nil) + (set-process-sentinel mu4e--server-process nil) + (let ((delete-exited-processes t)) + (mu4e--server-call-mu '(quit))) + ;; try sending SIGINT (C-c) to process, so it can exit gracefully + (ignore-errors + (signal-process proc 'SIGINT)))) + (setq + mu4e--server-process nil + mu4e--server-buf nil)) + +;; error codes are defined in src/mu-util +;;(defconst mu4e-xapian-empty 19 "Error code: xapian is empty/non-existent") + +(defun mu4e--server-sentinel (proc _msg) + "Function called when the server process PROC terminates with MSG." + (let ((status (process-status proc)) (code (process-exit-status proc))) + (mu4e-log 'misc "* famous last words from server: '%s'" mu4e--server-buf) + (setq mu4e--server-process nil) + (setq mu4e--server-buf "") ;; clear any half-received sexps + (cond + ((eq status 'signal) + (cond + ((or(eq code 9) (eq code 2)) (message nil)) + ;;(message "the mu server process has been stopped")) + (t (mu4e-error (format "server process received signal %d" code))))) + ((eq status 'exit) + (cond + ((eq code 0) + (message nil)) ;; don't do anything + ((eq code 11) + (error "schema mismatch; please re-init mu from command-line")) + ((eq code 19) + (error "mu database is locked by another process")) + (t (error "mu server process ended with exit code %d" code)))) + (t + (error "something bad happened to the mu server process"))))) + +(defun mu4e--server-call-mu (form) + "Call the mu server with some command FORM." + (unless (mu4e-running-p) (mu4e--server-start)) + (let* ((print-length nil) (print-level nil) + (cmd (format "%S" form))) + (mu4e-log 'to-server "%s" cmd) + (process-send-string mu4e--server-process (concat cmd "\n")))) + +(defun mu4e--server-add (path) + "Add the message at PATH to the database. +On success, we receive `'(:info add :path <path> :docid <docid>)' +as well as `'(:update <msg-sexp>)`'; otherwise, we receive an error." + (mu4e--server-call-mu `(add :path ,path))) + +(defun mu4e--server-contacts (personal after maxnum tstamp) + "Ask for contacts with PERSONAL AFTER MAXNUM TSTAMP. + +S-expression (:contacts (<list>) :tstamp \"<tstamp>\") +is expected in response. + +If PERSONAL is non-nil, only get personal contacts, if AFTER is +non-nil, get only contacts seen AFTER (the time_t value). If MAX is non-nil, +get at most MAX contacts." + (mu4e--server-call-mu + `(contacts + :personal ,(and personal t) + :after ,(or after nil) + :tstamp ,(or tstamp nil) + :maxnum ,(or maxnum nil)))) + +(defun mu4e--server-data (kind) + "Request data of some KIND. +KIND is a symbol. Currently supported kinds: maildirs." + (mu4e--server-call-mu + `(data :kind ,kind))) + +(defun mu4e--server-find (query threads sortfield sortdir maxnum skip-dups + include-related) + "Run QUERY with THREADS SORTFIELD SORTDIR MAXNUM SKIP-DUPS INCLUDE-RELATED. + +If THREADS is non-nil, show results in threaded fashion, +SORTFIELD is a symbol describing the field to sort by (or nil); +see `mu4e~headers-sortfield-choices'. If SORT is `descending', +sort Z->A, if it's `ascending', sort A->Z. MAXNUM determines the +maximum number of results to return, or nil for unlimited. If +SKIP-DUPS is non-nil, show only one of duplicate messages (see +`mu4e-headers-skip-duplicates'). If INCLUDE-RELATED is non-nil, +include messages related to the messages matching the search +query (see `mu4e-headers-include-related'). + +For each result found, a function is called, depending on the +kind of result. The variables `mu4e-error-func' contain the +function that to be be called for, resp., a message (header) +or an error." + (mu4e--server-call-mu + `(find + :query ,query + :threads ,(and threads t) + :sortfield ,sortfield + :descending ,(if (eq sortdir 'descending) t nil) + :maxnum ,maxnum + :skip-dups ,(and skip-dups t) + :include-related ,(and include-related t)))) + +(defun mu4e--server-index (&optional cleanup lazy-check) + "Index messages. +If CLEANUP is non-nil, remove messages which are in the database +but no longer in the filesystem. If LAZY-CHECK is non-nil, only +consider messages for which the time stamp (ctime) of the +directory they reside in has not changed since the previous +indexing run. This is much faster than the non-lazy check, but +won't update messages that have change (rather than having been +added or removed), since merely editing a message does not update +the directory time stamp." + (mu4e--server-call-mu + `(index :cleanup ,(and cleanup t) + :lazy-check ,(and lazy-check t)))) + +(defun mu4e--server-mkdir (path &optional update) + "Create a new maildir-directory at filesystem PATH. +When UPDATE is non-nil, send a update when completed. +PATH must be below the root-maildir." + ;; handle maildir cache + (if (not (string-prefix-p (mu4e-root-maildir) path)) + (mu4e-error "Cannot create maildir outside root-maildir") + (add-to-list 'mu4e-maildir-list ;; update cache + (substring path (length (mu4e-root-maildir))))) + (mu4e--server-call-mu `(mkdir + :path ,path + :update ,(or update nil)))) + +(defun mu4e--server-move (docid-or-msgid &optional maildir flags no-view) + "Move message identified by DOCID-OR-MSGID. +Optionally to MAILDIR and optionally setting FLAGS. If MAILDIR is +nil, message will be moved within the same maildir. + +At least one of MAILDIR and FLAGS must be specified. Note that +even when MAILDIR is nil, this is still a filesystem move, since +a change in flags implies a change in message filename. + +MAILDIR must be a maildir, that is, the part _without_ cur/ or new/ +or the root-maildir-prefix. E.g. \"/archive\". This directory must +already exist. + +The FLAGS parameter can have the following forms: + 1. a list of flags such as `(passed replied seen)' + 2. a string containing the one-char versions of the flags, e.g. \"PRS\" + 3. a delta-string specifying the changes with +/- and the one-char flags, + e.g. \"+S-N\" to set Seen and remove New. + +The flags are any of `deleted', `flagged', `new', `passed', `replied' `seen' or +`trashed', or the corresponding \"DFNPRST\" as defined in [1]. See +`mu4e-string-to-flags' and `mu4e-flags-to-string'. +The server reports the results for the operation through +`mu4e-update-func'. + +If the variable `mu4e-change-filenames-when-moving' is +non-nil, moving to a different maildir generates new names forq +the target files; this helps certain tools (such as mbsync). + +If NO-VIEW is non-nil, do not update the view. + +Returns either (:update ... ) or (:error ) sexp, which are handled my +`mu4e-update-func' and `mu4e-error-func', respectively." + (unless (or maildir flags) + (mu4e-error "At least one of maildir and flags must be specified")) + (unless (or (not maildir) + (file-exists-p + (mu4e-join-paths (mu4e-root-maildir) maildir))) + (mu4e-error "Target directory does not exist")) + (mu4e--server-call-mu + `(move + :docid ,(if (stringp docid-or-msgid) nil docid-or-msgid) + :msgid ,(if (stringp docid-or-msgid) docid-or-msgid nil) + :flags ,(or flags nil) + :maildir ,(or maildir nil) + :rename ,(and maildir mu4e-change-filenames-when-moving t) + :no-view ,(and no-view t)))) + +(defun mu4e--server-ping () + "Sends a ping to the mu server, expecting a (:pong ...) in response." + (mu4e--server-call-mu `(ping))) + +(defun mu4e--server-queries (queries) + "Sends queries to the mu server, expecting a (:queries ...) sexp in response. +QUERIES is a list of queries for the number of results with +read/unread status are returned in the pong-response." + (mu4e--server-call-mu `(queries :queries ,queries))) + +(defun mu4e--server-remove (docid-or-path) + "Remove message with either DOCID or PATH. +The results are reported through either (:update ... ) +or (:error) sexps." + (if (stringp docid-or-path) + (mu4e--server-call-mu `(remove :path ,docid-or-path)) + (mu4e--server-call-mu `(remove :docid ,docid-or-path)))) + +(defun mu4e--server-view (docid-or-msgid &optional mark-as-read) + "View a message referred to by DOCID-OR-MSGID. +Optionally, if MARK-AS-READ is non-nil, the backend marks the +message as \"read\" before returning, if not already. The result +will be delivered to the function registered as `mu4e-view-func'." + (mu4e--server-call-mu + `(view + :docid ,(if (stringp docid-or-msgid) nil docid-or-msgid) + :msgid ,(if (stringp docid-or-msgid) docid-or-msgid nil) + :mark-as-read ,(and mark-as-read t) + ;; when moving (due to mark-as-read), change filenames + ;; if so configured. Note: currently this *ignored* + ;; because mbsync seems to get confused. + :rename ,(and mu4e-change-filenames-when-moving t)))) + + +(provide 'mu4e-server) +;;; mu4e-server.el ends here diff --git a/mu4e/mu4e-speedbar.el b/mu4e/mu4e-speedbar.el new file mode 100644 index 0000000..c2c414e --- /dev/null +++ b/mu4e/mu4e-speedbar.el @@ -0,0 +1,133 @@ +;;; mu4e-speedbar --- Speedbar support for mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2012-2021 Antono Vasiljev, Dirk-Jan C. Binnema + +;; Author: Antono Vasiljev <self@antono.info> +;; Version: 0.1 +;; Keywords: file, tags, tools + +;; This file is not part of GNU Emacs. + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3, or (at your option) +;; any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: +;; +;; Speedbar provides a frame in which files, and locations in files +;; are displayed. These functions provide mu4e specific support, +;; showing maildir list in the side-bar. +;; +;; This file requires speedbar. + +;;; Code: + +(require 'speedbar) +(require 'mu4e-vars) +(require 'mu4e-headers) +(require 'mu4e-context) +(require 'mu4e-bookmarks) + +(defvar mu4e-main-speedbar-key-map nil + "Keymap used when in mu4e display mode.") +(defvar mu4e-headers-speedbar-key-map nil + "Keymap used when in mu4e display mode.") +(defvar mu4e-view-speedbar-key-map nil + "Keymap used when in mu4e display mode.") + +(defvar mu4e-main-speedbar-menu-items nil + "Additional menu-items to add to speedbar frame.") +(defvar mu4e-headers-speedbar-menu-items nil + "Additional menu-items to add to speedbar frame.") +(defvar mu4e-view-speedbar-menu-items nil + "Additional menu-items to add to speedbar frame.") + + +(defun mu4e-speedbar-install-variables () + "Install those variables used by speedbar to enhance mu4e." + (add-hook 'mu4e-context-changed-hook + #'mu4e~speedbar-context-changed-hook-fn) + (dolist (keymap + '( mu4e-main-speedbar-key-map + mu4e-headers-speedbar-key-map + mu4e-view-speedbar-key-map)) + (unless keymap + (setq keymap (speedbar-make-specialized-keymap)) + (define-key keymap "RET" 'speedbar-edit-line) + (define-key keymap "e" 'speedbar-edit-line)))) + +(defun mu4e~speedbar-context-changed-hook-fn () + (when (buffer-live-p speedbar-buffer) + (with-current-buffer speedbar-buffer + (let ((inhibit-read-only t)) + (mu4e-speedbar-buttons))))) + +(with-eval-after-load 'speedbar + (mu4e-speedbar-install-variables)) + +(defun mu4e~speedbar-render-maildir-list () + "Insert the list of maildirs in the speedbar." + (interactive) + (when (buffer-live-p speedbar-buffer) + (with-current-buffer speedbar-buffer + (mapcar (lambda (maildir-name) + (speedbar-insert-button + (concat " " maildir-name) + 'mu4e-highlight-face + 'highlight + 'mu4e~speedbar-maildir + maildir-name)) + (mu4e-get-maildirs))))) + +(defun mu4e~speedbar-maildir (&optional _text token _ident) + "Jump to maildir TOKEN. TEXT and INDENT are not used." + (dframe-with-attached-buffer + (mu4e-search (concat "\"maildir:" token "\"") current-prefix-arg))) + +(defun mu4e~speedbar-render-bookmark-list () + "Insert the list of bookmarks in the speedbar" + (interactive) + (mapcar (lambda (bookmark) + (unless (plist-get bookmark :hide) + (speedbar-insert-button + (concat " " (plist-get bookmark :name)) + 'mu4e-highlight-face + 'highlight + 'mu4e~speedbar-bookmark + (plist-get bookmark :query)))) + (mu4e-bookmarks))) + +(defun mu4e~speedbar-bookmark (&optional _text token _ident) + "Run bookmarked query TOKEN. TEXT and INDENT are not used." + (dframe-with-attached-buffer + (mu4e-search token current-prefix-arg))) + +;;;###autoload +(defun mu4e-speedbar-buttons (&optional _buffer) + "Create buttons for any mu4e BUFFER." + (interactive) + (erase-buffer) + (insert (propertize "* mu4e\n\n" 'face 'mu4e-title-face)) + + (insert (propertize " Bookmarks\n" 'face 'mu4e-title-face)) + (mu4e~speedbar-render-bookmark-list) + (insert "\n") + (insert (propertize " Maildirs\n" 'face 'mu4e-title-face)) + (mu4e~speedbar-render-maildir-list)) + +(defun mu4e-main-speedbar-buttons (buffer) (mu4e-speedbar-buttons buffer)) +(defun mu4e-headers-speedbar-buttons (buffer) (mu4e-speedbar-buttons buffer)) +(defun mu4e-view-speedbar-buttons (buffer) (mu4e-speedbar-buttons buffer)) + +;;; _ +(provide 'mu4e-speedbar) +;;; mu4e-speedbar.el ends here diff --git a/mu4e/mu4e-thread.el b/mu4e/mu4e-thread.el new file mode 100644 index 0000000..c973745 --- /dev/null +++ b/mu4e/mu4e-thread.el @@ -0,0 +1,295 @@ +;;; mu4e-thread.el --- Thread folding support -*- lexical-binding: t -*- + +;; Copyright (C) 2023 Nicolas P. Rougier + +;; Author: Nicolas P. Rougier <Nicolas.Rougier@inria.fr> +;; Keywords: mail + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; mu4e-thread.el is a library that allows to fold and unfold threads in mu4e +;; headers mode. Folding works by creating an overlay over thread children that +;; display a summary (number of hidden messages and possibly number of unread +;; messages). + +;; Folding is performed just-in-time such that it is quite fast to +;; fold/unfold threads. When a thread has unread messages, the folding stops at +;; the first unread message unless `mu4e-thread-fold-unread` has been set to t. + +;; Similarly, when a thread has marked messages, the folding stops at the first +;; marked message. + +;; Note, you can only use these functions when threads are available, roughly +;; when `mu4e-search-threads' in non-nil. + +;;; Usage example: +;; +;; After a search, mu4e-thread-mode will be enable when threads +;; are available; so, to automatically sort them: +;; (add-hook 'mu4e-thread-mode-hook #'mu4e-thread-fold-apply-all) + +;;; Code: + +(require 'mu4e-vars) +(require 'mu4e-message) +(require 'mu4e-mark) + +(defcustom mu4e-thread-fold-unread nil + "Whether to fold unread messages in a thread." + :type 'boolean + :group 'mu4e-headers) + +(defcustom mu4e-thread-fold-single-children nil + "When non-nil, fold a thread even if there is only a single child. +Otherwise, do not not fold single children since would simply +hide the single child." + :type 'boolean + :group 'mu4e-headers) + +(defface mu4e-thread-fold-face + `((t :inherit mu4e-highlight-face)) + "Face for the information line of a folded thread." + :group 'mu4e-faces) + +(defvar-local mu4e-thread--fold-status nil + "Global folding status.") + +(defvar-local mu4e-thread--docids nil + "Thread list whose folding has been set individually.") + +(defvar mu4e-headers-fields) ;; defined in mu4e-headers.el +(defun mu4e-thread-fold-info (count unread) + "Text to be displayed for a folded thread. +There are COUNT hidden and UNREAD messages overall." + (let ((size (+ 2 (apply #'+ (mapcar (lambda (item) (or (cdr item) 0)) + mu4e-headers-fields)))) + (msg (concat (format"[%d hidden messages%s]\n" count + (if (> unread 0) + (format ", %d unread" unread) + ""))))) + (propertize (concat " " (make-string size ?•) " " msg)))) + +(defun mu4e-thread-message-folded-p () + "Is point in a folded area?" + (when-let* ((overlay (mu4e-thread-is-folded)) + (beg (overlay-start overlay)) + (end (overlay-end overlay))) + (and (>= (point) beg) (< (point) end)))) + +(declare-function 'mu4e~headers-thread-root-p "mu4e-headers") +(defalias 'mu4e-thread-is-root 'mu4e~headers-thread-root-p) + +(defun mu4e-thread-goto-root () + "Go to the root of the current thread." + (interactive) + (goto-char (mu4e-thread-root)) + (beginning-of-line)) + +(defun mu4e-thread-root () + "Get the root of the current thread." + (interactive) + (let ((point)) + (save-excursion + (while (and (not (bobp)) + (not (mu4e-thread-is-root))) + (forward-line -1)) + (setq point (point))) + point)) + +(declare-function 'mu4e-headers-prev-thread "mu4e-headers") +(declare-function 'mu4e-headers-next-thread "mu4e-headers") + +(defalias 'mu4e-thread-goto-prev 'mu4e-headers-prev-thread) +(defalias 'mu4e-thread-goto-next 'mu4e-headers-next-thread) + +(defun mu4e-thread-prev () + "Get the root of the previous thread (if any)." + (save-excursion + (when (mu4e-thread-goto-prev) + (mu4e-thread-root)))) + +(defun mu4e-thread-next() + "Get the root of the next thread (if any)." + (save-excursion + (when (mu4e-thread-goto-next) + (mu4e-thread-root)))) + +(defun mu4e-thread-is-folded () + "Test if thread at point is folded." + (interactive) + (let* ((thread-beg (mu4e-thread-root)) + (thread-end (or (mu4e-thread-next) (point-max))) + (overlays (overlays-in thread-beg thread-end))) + (catch 'folded + (dolist (overlay overlays) + (when (overlay-get overlay 'mu4e-thread-folded) + (throw 'folded overlay)))))) + +(defun mu4e-thread-fold-toggle-all () + "Toggle all threads folding unconditionally. +Reset individual folding states." + (interactive) + (setq mu4e-thread--docids nil) + (if mu4e-thread--fold-status + (mu4e-thread-unfold-all) + (mu4e-thread-fold-all))) + +(defun mu4e-thread-fold-apply-all () + "Apply global folding status to all threads not set individually." + (interactive) + ;; Global fold status + (if mu4e-thread--fold-status + (mu4e-thread-fold-all) + (mu4e-thread-unfold-all)) + ;; Individual fold status + (save-excursion + (goto-char (point-min)) + (catch 'end-search + (while (not (eobp)) + (when-let* ((msg (get-text-property (point) 'msg)) + (docid (mu4e-message-field msg :docid)) + (state (cdr (assoc docid mu4e-thread--docids)))) + (if (eq state 'folded) + (mu4e-thread-fold) + (mu4e-thread-unfold))) + (unless (mu4e-thread-next) + (throw 'end-search t)) + (mu4e-thread-goto-next))))) + +(defun mu4e-thread-fold-all () + "Fold all threads unconditionally." + (interactive) + (setq mu4e-thread--fold-status t) + + (save-excursion + (goto-char (point-min)) + (catch 'done + (while (not (eobp)) + (mu4e-thread-fold t) + (unless (mu4e-thread-goto-next) + (throw 'done t)))))) + +(defun mu4e-thread-unfold-all () + "Unfold all threads unconditionally." + (interactive) + (setq mu4e-thread--fold-status nil) + (remove-overlays (point-min) (point-max) 'mu4e-thread-folded t)) + +(defun mu4e-thread-fold-toggle () + "Toggle folding for thread at point." + (interactive) + (if (mu4e-thread-is-folded) + (mu4e-thread-unfold) + (mu4e-thread-fold))) + +(defun mu4e-thread-fold-toggle-goto-next () + "Toggle folding for thread at point and go to next thread." + (interactive) + (if (mu4e-thread-is-folded) + (mu4e-thread-unfold-goto-next) + (mu4e-thread-fold-goto-next))) + +(defun mu4e-thread-unfold (&optional no-save) + "Unfold thread at point and store state unless NO-SAVE is t." + (interactive) + (unless (eq (line-end-position) (point-max)) + (when-let ((overlay (mu4e-thread-is-folded))) + (unless no-save + (mu4e-thread--save-state 'unfolded)) + (delete-overlay overlay)))) + +(defun mu4e-thread--save-state (state) + "Save the folding STATE of thread at point." + (save-excursion + (mu4e-thread-goto-root) + (when-let* ((msg (get-text-property (point) 'msg)) + (docid (mu4e-message-field msg :docid))) + (setf (alist-get docid mu4e-thread--docids) state)))) + +(defun mu4e-thread-fold (&optional no-save) + "Fold thread at point and store state unless NO-SAVE is t." + (interactive) + (unless (eq (line-end-position) (point-max)) + (let* ((thread-beg (mu4e-thread-root)) + (thread-end (mu4e-thread-next)) + (thread-end (if thread-end (1- thread-end) (point-max))) + (unread-count 0) + (fold-beg (save-excursion + (goto-char thread-beg) + (forward-line) + (point))) + (fold-end (save-excursion + (goto-char thread-beg) + (forward-line) + (catch 'fold-end + (while (and (not (eobp)) + (get-text-property (point) 'msg) + (and thread-end (< (point) thread-end))) + (let* ((msg (get-text-property (point) 'msg)) + (docid (mu4e-message-field msg :docid)) + (flags (mu4e-message-field msg :flags)) + (unread (memq 'unread flags))) + (when (mu4e-mark-docid-marked-p docid) + (throw 'fold-end (point))) + (when unread + (unless mu4e-thread-fold-unread + (throw 'fold-end (point))) + (setq unread-count (+ 1 unread-count)))) + (forward-line))) + (point)))) + (unless no-save + (mu4e-thread--save-state 'folded)) + (let ((child-count (count-lines fold-beg fold-end)) + (unread-count (if mu4e-thread-fold-unread unread-count 0))) + (when (> child-count (if mu4e-thread-fold-single-children 0 1)) + (let ((inhibit-read-only t) + (overlay (make-overlay fold-beg fold-end)) + (info (mu4e-thread-fold-info child-count unread-count))) + (add-text-properties fold-beg (+ fold-beg 1) + '(face mu4e-thread-fold-face)) + (overlay-put overlay 'mu4e-thread-folded t) + (overlay-put overlay 'display info))))))) + +(defun mu4e-thread-fold-goto-next () + "Fold the thread at point and go to next thread." + (interactive) + (unless (eq (line-end-position) (point-max)) + (mu4e-thread-fold) + (mu4e-thread-goto-next))) + +(defun mu4e-thread-unfold-goto-next () + "Unfold the thread at point and go to next thread." + (interactive) + (unless (eq (line-end-position) (point-max)) + (mu4e-thread-unfold) + (mu4e-thread-goto-next))) + +(define-minor-mode mu4e-thread-mode + "Mode for thread-support." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap + (let ((map (make-sparse-keymap))) + (define-key map (kbd "<S-left>") #'mu4e-thread-goto-root) + (define-key map (kbd "<tab>") #'mu4e-thread-fold-toggle-goto-next) + (define-key map (kbd "<C-tab>") #'mu4e-thread-fold-toggle-goto-next) + (define-key map (kbd "<backtab>") #'mu4e-thread-fold-toggle-all) + map)) + +(provide 'mu4e-thread) +;;; mu4e-thread.el ends here diff --git a/mu4e/mu4e-update.el b/mu4e/mu4e-update.el new file mode 100644 index 0000000..5bb9e1d --- /dev/null +++ b/mu4e/mu4e-update.el @@ -0,0 +1,335 @@ +;;; mu4e-update.el --- Update the mu4e message store -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Updating the mu4e message store: calling a mail retrieval program and +;; re-running the index. + +;;; Code: + +(require 'mu4e-helpers) +(require 'mu4e-server) + +;;; Customization + +(defcustom mu4e-get-mail-command "true" + "Shell command for retrieving new mail. +Common values are \"offlineimap\", \"fetchmail\" or \"mbsync\", but +arbitrary shell-commands can be used. + +When set to the literal string \"true\" (the default), the +command simply finishes successfully (running the \"true\" +command) without retrieving any mail. This can be useful when +mail is already retrieved in another way, such as a local MDA." + :type 'string + :group 'mu4e + :safe 'stringp) + +(defcustom mu4e-index-update-error-warning t + "Whether to display warnings during the retrieval process. +This depends on the `mu4e-get-mail-command' exit code." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-index-update-error-continue t + "Whether to continue with indexing after an error during retrieval." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-index-update-in-background t + "Whether to retrieve mail in the background." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-index-cleanup t + "Whether to run a cleanup phase after indexing. + +That is, validate that each message in the message store has a +corresponding message file in the filesystem. + +Having this option as t ensures that no non-existing messages are +shown but can slow with large message stores on slow file-systems." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-index-lazy-check nil + "Whether to only use a \"lazy\" check during reindexing. +This influences how we decide whether a message +needs (re)indexing or not. + +When this is set to non-nil, mu only uses the directory +timestamps to decide whether it needs to check the messages +beneath it. This makes indexing much faster, but might miss some +changes. For this, you might want to occasionally call +`mu4e-update-index-nonlazy'; `mu4e-update-pre-hook' can be used +to automate this." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-update-interval nil + "Number of seconds between mail retrieval/indexing. +If nil, don't update automatically. Note, changes in +`mu4e-update-interval' only take effect after restarting mu4e. + +Important, the automatic update *only* works when `mu4e' is +running." + :type '(choice (const :tag "No automatic update" nil) + (integer :tag "Seconds")) + :group 'mu4e + :safe 'integerp) + +(defvar mu4e-update-pre-hook nil + "Hook run just *before* the mail-retrieval / database updating process starts. +You can use this hook for example to `mu4e-get-mail-command' with +some specific setting.") + +(defcustom mu4e-hide-index-messages nil + "Whether to hide the \"Indexing...\" and contacts messages." + :type 'boolean + :group 'mu4e) + +(defvar mu4e-index-updated-hook nil + "Hook run when the indexing process has completed. +The variable `mu4e-index-update-status' can be used to get +information about what changed.") + +(defvar mu4e-message-changed-hook nil + "Hook run when there is a message changed in the data store. +For new messages, it depends on `mu4e-index-updated-hook'. This +can be used as a simple way to invoke some action when a message +changed") + +(defvar mu4e-index-update-status nil + "Last-seen completed update status, based on server status messages. + +If non-nil, this is a plist of the form: +\( +:checked <number of messages processed> (checked whether up-to-date) +:updated <number of messages updated/added +:cleaned-up <number of stale messages removed from store +:stamp <emacs (current-time) timestamp for the status)") + +(defconst mu4e-last-update-buffer "*mu4e-last-update*" + "Name of buffer with cloned from the last update buffer. +Useful for diagnosing update problems.") + + +;;; Internal variables / const +(defconst mu4e--update-name " *mu4e-update*" + "Name of the process and buffer to update mail.") +(defvar mu4e--progress-reporter nil + "Internal, the progress reporter object.") +(defvar mu4e--update-timer nil + "The mu4e update timer.") +(defconst mu4e--update-buffer-height 8 + "Height of the mu4e message retrieval/update buffer.") +(defvar mu4e--get-mail-ask-password "mu4e get-mail: Enter password: " + "Query string for `mu4e-get-mail-command' password.") +(defvar mu4e--get-mail-password-regexp "^Remote: Enter password: $" + "Regexp for a `mu4e-get-mail-command' password query.") + + +(defun mu4e--get-mail-process-filter (proc msg) + "Filter the MSG output of the `mu4e-get-mail-command' PROC. + +Currently the filter only checks if the command asks for a +password by matching the output against +`mu4e~get-mail-password-regexp'. The messages are inserted into +the process buffer. + +Also scrolls to the final line, and update the progress +throbber." + (when mu4e--progress-reporter + (progress-reporter-update mu4e--progress-reporter)) + + (when (string-match mu4e--get-mail-password-regexp msg) + (if (process-get proc 'x-interactive) + (process-send-string proc + (concat (read-passwd mu4e--get-mail-ask-password) + "\n")) + ;; TODO kill process? + (mu4e-error "Unrecognized password request"))) + (when (process-buffer proc) + (let ((inhibit-read-only t) + (procwin (get-buffer-window (process-buffer proc)))) + ;; Insert at end of buffer. Leave point alone. + (with-current-buffer (process-buffer proc) + (goto-char (point-max)) + (if (string-match ".*\r\\(.*\\)" msg) + (progn + ;; kill even with \r + (end-of-line) + (let ((end (point))) + (beginning-of-line) + (delete-region (point) end)) + (insert (match-string 1 msg))) + (insert msg))) + ;; Auto-scroll unless user is interacting with the window. + (when (and (window-live-p procwin) + (not (eq (selected-window) procwin))) + (with-selected-window procwin + (goto-char (point-max))))))) + +(defun mu4e-index-message (frm &rest args) + "Display FRM with ARGS like `mu4e-message' for index messages. +However, if `mu4e-hide-index-messages' is non-nil, do not display anything." + (unless mu4e-hide-index-messages + (apply 'mu4e-message frm args))) + +(defun mu4e-update-index () + "Update the mu4e index." + (interactive) + (mu4e--server-index mu4e-index-cleanup mu4e-index-lazy-check)) + +(defun mu4e-update-index-nonlazy () + "Update the mu4e index non-lazily. +This is just a convenience wrapper for indexing the non-lazy way +if you otherwise want to use `mu4e-index-lazy-check'." + (interactive) + (let ((mu4e-index-cleanup t) (mu4e-index-lazy-check nil)) + (mu4e-update-index))) + +(defvar mu4e--update-buffer nil + "The buffer of the update process when updating.") + +(define-derived-mode mu4e--update-mail-mode special-mode "mu4e:update" + "Major mode used for retrieving new e-mail messages in `mu4e'.") + +(define-key mu4e--update-mail-mode-map (kbd "q") 'mu4e-kill-update-mail) + +(defun mu4e--temp-window (buf height) + "Create a temporary window with HEIGHT at the bottom BUF. + +This function uses `display-buffer' with a default preset. + +To override this behavior, customize `display-buffer-alist'." + (display-buffer buf `(display-buffer-at-bottom + (preserve-size . (nil . t)) + (height . ,height) + (inhibit-same-window . t) + (window-height . fit-window-to-buffer))) + (set-window-buffer (get-buffer-window buf) buf)) + +(defun mu4e--update-sentinel-func (proc _msg) + "Sentinel function for the update process PROC." + (when mu4e--progress-reporter + (progress-reporter-done mu4e--progress-reporter) + (setq mu4e--progress-reporter nil)) + (unless mu4e-hide-index-messages + (message nil)) + (if (or (not (eq (process-status proc) 'exit)) + (/= (process-exit-status proc) 0)) + (progn + (when mu4e-index-update-error-warning + (mu4e-message "Update process returned with non-zero exit code") + (sit-for 5)) + (when mu4e-index-update-error-continue + (mu4e-update-index))) + (mu4e-update-index)) + (when (buffer-live-p mu4e--update-buffer) + (delete-windows-on mu4e--update-buffer) + ;; clone the update buffer for diagnosis + (when (get-buffer mu4e-last-update-buffer) + (kill-buffer mu4e-last-update-buffer)) + (with-current-buffer mu4e--update-buffer + (special-mode) + (clone-buffer mu4e-last-update-buffer)) + ;; and kill the buffer itself; the cloning is needed + ;; so the temp window handling works as expected. + (kill-buffer mu4e--update-buffer))) + +;; complicated function, as it: +;; - needs to check for errors +;; - (optionally) pop-up a window +;; - (optionally) check password requests +(defun mu4e--update-mail-and-index-real (run-in-background) + "Get a new mail by running `mu4e-get-mail-command'. +If +RUN-IN-BACKGROUND is non-nil (or called with prefix-argument), +run in the background; otherwise, pop up a window." + (let* ((process-connection-type t) + (proc (start-process-shell-command + mu4e--update-name mu4e--update-name + mu4e-get-mail-command)) + (buf (process-buffer proc)) + (win (or run-in-background + (mu4e--temp-window buf mu4e--update-buffer-height)))) + (set-process-query-on-exit-flag proc nil) + (setq mu4e--update-buffer buf) + (when (window-live-p win) + (with-selected-window win + (erase-buffer) + (insert "\n") ;; FIXME -- needed so output starts + (mu4e--update-mail-mode))) + (setq mu4e--progress-reporter + (unless mu4e-hide-index-messages + (make-progress-reporter + (mu4e-format "Retrieving mail...")))) + (set-process-sentinel proc 'mu4e--update-sentinel-func) + ;; if we're running in the foreground, handle password requests + (unless run-in-background + (process-put proc 'x-interactive (not run-in-background)) + (set-process-filter proc 'mu4e--get-mail-process-filter)))) + +(defun mu4e-update-mail-and-index (run-in-background) + "Retrieve new mail by running `mu4e-get-mail-command'. +If RUN-IN-BACKGROUND is non-nil (or called with prefix-argument), +run in the background; otherwise, pop up a window." + (interactive "P") + (unless mu4e-get-mail-command + (mu4e-error "`mu4e-get-mail-command' is not defined")) + (if (and (buffer-live-p mu4e--update-buffer) + (process-live-p (get-buffer-process mu4e--update-buffer))) + (mu4e-message "Update process is already running") + (progn + (run-hooks 'mu4e-update-pre-hook) + (mu4e--update-mail-and-index-real run-in-background)))) + +(defun mu4e-kill-update-mail () + "Stop the update process by killing it." + (interactive) + (let* ((proc (and (buffer-live-p mu4e--update-buffer) + (get-buffer-process mu4e--update-buffer)))) + (when (process-live-p proc) + (kill-process proc t)))) + +(define-minor-mode mu4e-update-minor-mode + "Mode for triggering mu4e updates." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap + (let ((map (make-sparse-keymap))) + (define-key map (kbd "C-S-u") #'mu4e-update-mail-and-index) + ;; for terminal users + (define-key map (kbd "C-c C-u") #'mu4e-update-mail-and-index) + map)) + +(provide 'mu4e-update) +;;; mu4e-update.el ends here diff --git a/mu4e/mu4e-vars.el b/mu4e/mu4e-vars.el new file mode 100644 index 0000000..6a95c32 --- /dev/null +++ b/mu4e/mu4e-vars.el @@ -0,0 +1,392 @@ +;;; mu4e-vars.el --- Variables and faces for mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: + +(require 'message) +(require 'mu4e-helpers) + +;;; Configuration +(defgroup mu4e nil + "Mu4e - an email-client for Emacs." + :group 'mail) + +(defcustom mu4e-confirm-quit t + "Whether to confirm to quit mu4e." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-modeline-support t + "Support for showing information in the modeline." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-notification-support nil + "Support for new-message notifications." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-org-support t + "Support Org-mode links." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-speedbar-support nil + "Support having a speedbar to navigate folders/bookmarks." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-eldoc-support nil + "Support eldoc help in the headers-view." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-date-format-long "%c" + "Date format to use in the message view. +Follows the format of `format-time-string'." + :type 'string + :group 'mu4e) + +(defcustom mu4e-dim-when-loading t + "Dim buffer text when loading new data. +If non-nil, dim some buffers during data retrieval and rendering, +and show some \"Loading\" banner." + :type 'boolean + :group 'mu4e) + + +;;; Faces + +(defgroup mu4e-faces nil + "Type faces (fonts) used in mu4e." + :group 'mu4e + :group 'faces) + +(defface mu4e-unread-face + '((t :inherit font-lock-keyword-face :weight bold)) + "Face for an unread message header." + :group 'mu4e-faces) + +(defface mu4e-trashed-face + '((t :inherit font-lock-comment-face :strike-through t)) + "Face for an message header in the trash folder." + :group 'mu4e-faces) + +(defface mu4e-draft-face + '((t :inherit font-lock-string-face)) + "Face for a draft message header. +I.e. a message with the draft flag set." + :group 'mu4e-faces) + +(defface mu4e-flagged-face + '((t :inherit font-lock-constant-face :weight bold)) + "Face for a flagged message header." + :group 'mu4e-faces) + +(defface mu4e-replied-face + '((t :inherit font-lock-builtin-face :weight normal :slant normal)) + "Face for a replied message header." + :group 'mu4e-faces) + +(defface mu4e-forwarded-face + '((t :inherit font-lock-builtin-face :weight normal :slant normal)) + "Face for a passed (forwarded) message header." + :group 'mu4e-faces) + +(defface mu4e-header-face + '((t :inherit default)) + "Face for a header without any special flags." + :group 'mu4e-faces) + +(defface mu4e-related-face + '((t :inherit default :slant italic)) + "Face for a \='related' header." :group 'mu4e-faces) + +(defface mu4e-header-title-face + '((t :inherit font-lock-type-face)) + "Face for a header title in the headers view." + :group 'mu4e-faces) + +(defface mu4e-header-highlight-face + `((t :inherit hl-line :weight bold :underline t + ,@(and (>= emacs-major-version 27) '(:extend t)))) + "Face for the header at point." + :group 'mu4e-faces) + +(defface mu4e-header-marks-face + '((t :inherit font-lock-preprocessor-face)) + "Face for the mark in the headers list." + :group 'mu4e-faces) + +(defface mu4e-header-key-face + '((t :inherit message-header-name :weight bold)) + "Face used to highlight items in various places." + :group 'mu4e-faces) + +(defface mu4e-header-field-face + '((t :weight bold)) + "Face for a header field name (such as \"Subject:\" in \"Subject:\ +Foo\")." + :group 'mu4e-faces) + +(defface mu4e-header-value-face + '((t :inherit font-lock-type-face)) + "Face for a header value (such as \"Re: Hello!\")." + :group 'mu4e-faces) + +(defface mu4e-special-header-value-face + '((t :inherit font-lock-builtin-face)) + "Face for special header values." + :group 'mu4e-faces) + +(defface mu4e-link-face + '((t :inherit link)) + "Face for showing URLs and attachments in the message view." + :group 'mu4e-faces) + +(defface mu4e-contact-face + '((t :inherit font-lock-variable-name-face)) + "Face for showing URLs and attachments in the message view." + :group 'mu4e-faces) + +(defface mu4e-highlight-face + '((t :inherit highlight)) + "Face for highlighting things." + :group 'mu4e-faces) + +(defface mu4e-title-face + '((t :inherit font-lock-type-face :weight bold)) + "Face for a header title in the headers view." + :group 'mu4e-faces) + +(defface mu4e-modeline-face + '((t :inherit font-lock-string-face :weight bold)) + "Face for the query in the mode-line." + :group 'mu4e-faces) + +(defface mu4e-footer-face + '((t :inherit font-lock-comment-face)) + "Face for message footers (signatures)." + :group 'mu4e-faces) + +(defface mu4e-url-number-face + '((t :inherit font-lock-constant-face :weight bold)) + "Face for the number tags for URLs." + :group 'mu4e-faces) + +(defface mu4e-system-face + '((t :inherit font-lock-comment-face :slant italic)) + "Face for system message (such as the footers for message headers)." + :group 'mu4e-faces) + +(defface mu4e-ok-face + '((t :inherit font-lock-comment-face :weight bold :slant normal)) + "Face for things that are okay." + :group 'mu4e-faces) + +(defface mu4e-warning-face + '((t :inherit font-lock-warning-face :weight bold :slant normal)) + "Face for warnings / error." + :group 'mu4e-faces) + +(defface mu4e-compose-separator-face + '((t :inherit message-separator :slant italic)) + "Face for the headers/message separator in mu4e-compose-mode." + :group 'mu4e-faces) + +(defface mu4e-region-code + '((t (:background "DarkSlateGray"))) + "Face for highlighting marked region in mu4e-view buffer." + :group 'mu4e-faces) + +;;; Header information + +(defconst mu4e-header-info + '((:bcc + . (:name "Bcc" + :shortname "Bcc" + :help "Blind Carbon-Copy recipients for the message" + :sortable t)) + (:cc + . (:name "Cc" + :shortname "Cc" + :help "Carbon-Copy recipients for the message" + :sortable t)) + (:changed + . (:name "Changed" + :shortname "Chg" + :help "Date/time when the message was changed most recently" + :sortable t)) + (:date + . (:name "Date" + :shortname "Date" + :help "Date/time when the message was sent" + :sortable t)) + (:human-date + . (:name "Date" + :shortname "Date" + :help "Date/time when the message was sent" + :sortable :date)) + (:flags + . (:name "Flags" + :shortname "Flgs" + :help "Flags for the message" + :sortable nil)) + (:from + . (:name "From" + :shortname "From" + :help "The sender of the message" + :sortable t)) + (:from-or-to + . (:name "From/To" + :shortname "From/To" + :help "Sender of the message if it's not me; otherwise the recipient" + :sortable nil)) + (:maildir + . (:name "Maildir" + :shortname "Maildir" + :help "Maildir for this message" + :sortable t)) + (:list + . (:name "List-Id" + :shortname "List" + :help "Mailing list id for this message" + :sortable t)) + (:mailing-list + . (:name "List" + :shortname "List" + :help "Mailing list friendly name for this message" + :sortable :list)) + (:message-id + . (:name "Message-Id" + :shortname "MsgID" + :help "Message-Id for this message" + :sortable nil)) + (:path + . (:name "Path" + :shortname "Path" + :help "Full filesystem path to the message" + :sortable t)) + (:size + . (:name "Size" + :shortname "Size" + :help "Size of the message" + :sortable t)) + (:subject + . (:name "Subject" + :shortname "Subject" + :help "Subject of the message" + :sortable t)) + (:tags + . (:name "Tags" + :shortname "Tags" + :help "Tags for the message" + ;; sort by _first_ tag. + :sortable t)) + (:thread-subject + . (:name "Subject" + :shortname "Subject" + :help "Subject of the thread" + :sortable :subject)) + (:to + . (:name "To" + :shortname "To" + :help "Recipient of the message" + :sortable t))) + + "An alist of all possible header fields and information about them. + +This is used in the user-interface (the column headers in the +header list, and the fields the message view). + +Most fields should be self-explanatory. A special one is +`:from-or-to', which is equal to `:from' unless `:from' matches +one of the addresses in `(mu4e-personal-addresses)', in which +case it will be equal to `:to'. + +Furthermore, the property `:sortable' determines whether we can +sort by this field. This can be either a boolean (nil or t), or a +symbol for /another/ field. For example, the `:human-date' field +uses `:date' for that. + +Note, `:sortable' is not supported for custom header fields.") + +(defvar mu4e-header-info-custom + '( + ;; some examples & debug helpers. + + (:thread-path + . ;; Shows the internal thread-path + ( :name "Thread-path" + :shortname "Thp" + :help "The thread-path" + :function (lambda (msg) + (let ((thread (mu4e-message-field msg :thread))) + (or (and thread (plist-get thread :path)) ""))))) + + (:thread-date + . ;; Shows the internal thread-date + ( :name "Thread-date" + :shortname "Thd" + :help "The thread-date" + :function (lambda (msg) + (let* ((thread (mu4e-message-field msg :thread)) + (tdate (and thread (plist-get thread :date-tstamp)))) + (format-time-string "%F %T " (or tdate 0)))))) + (:recipnum + . + ( :name "Number of recipients" + :shortname "Recip#" + :help "Number of recipients for this message" + :function + (lambda (msg) + (format "%d" + (+ (length (mu4e-message-field msg :to)) + (length (mu4e-message-field msg :cc)))))))) + + "An alist of custom (user-defined) headers. +The format is similar to `mu4e-header-info', but adds a :function +property, which should point to a function that takes a message +plist as argument, and returns a string. See the default value of +`mu4e-header-info-custom for an example. + +Note that when using the gnus-based view, you only have access to +a limited set of message fields: only the ones used in the +header-view, not including, for instance, the message body.") + +;;; Internals + +(defvar-local mu4e~headers-view-win nil + "The view window connected to this headers view.") + +;; It's useful to have the current view message available to +;; `mu4e-view-mode-hooks' functions, and we set up this variable +;; before calling `mu4e-view-mode'. However, changing the major mode +;; clobbers any local variables. Work around that by declaring the +;; variable permanent-local. +(defvar-local mu4e--view-message nil "The message being viewed in view mode.") +(put 'mu4e--view-message 'permanent-local t) +;;; _ +(provide 'mu4e-vars) +;;; mu4e-vars.el ends here diff --git a/mu4e/mu4e-view.el b/mu4e/mu4e-view.el new file mode 100644 index 0000000..033540a --- /dev/null +++ b/mu4e/mu4e-view.el @@ -0,0 +1,1167 @@ +;;; mu4e-view.el --- Mode for viewing e-mail messages -*- lexical-binding: t -*- + +;; Copyright (C) 2021-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; In this file we define mu4e-view-mode (+ helper functions), which is used for +;; viewing e-mail messages + +;;; Code: + +(require 'cl-lib) +(require 'calendar) +(require 'gnus-art) +(require 'comint) +(require 'browse-url) +(require 'button) +(require 'epa) +(require 'epg) +(require 'thingatpt) + +(require 'mu4e-actions) +(require 'mu4e-compose) +(require 'mu4e-context) +(require 'mu4e-headers) +(require 'mu4e-mark) +(require 'mu4e-message) +(require 'mu4e-server) +(require 'mu4e-search) +(require 'mu4e-mime-parts) + +;; utility functions +(require 'mu4e-contacts) +(require 'mu4e-vars) + +;;; Options + +(defcustom mu4e-view-scroll-to-next t + "Move to the next message with `mu4e-view-scroll-up-or-next'. +When at the end of a message, move to the next one, if any. +Otherwise, don't move to the next message." + :type 'boolean + :group 'mu4e-view) + +(defcustom mu4e-view-fields + '(:from :to :cc :subject :flags :date :maildir :mailing-list :tags) + "Header fields to display in the message view buffer. + +For the complete list of available headers, see +`mu4e-header-info'. + +Note, you can use this to add fields that are not otherwise +shown; you can further tweak the other fields using e.g., +`gnus-visible-headers' and `gnus-ignored-headers' - see the gnus +documentation for details." + :type '(repeat symbol) + :group 'mu4e-view) + +(defcustom mu4e-view-actions + (delq nil `(("capture message" . mu4e-action-capture-message) + ("view in browser" . mu4e-action-view-in-browser) + ("browse online archive" . mu4e-action-browse-list-archive) + ,(when (fboundp 'xwidget-webkit-browse-url) + '("xview in xwidget" . mu4e-action-view-in-xwidget)) + ("show this thread" . mu4e-action-show-thread))) + "List of actions to perform on messages in view mode. +The actions are cons-cells of the form: + (NAME . FUNC) +where: +* NAME is the name of the action (e.g. \"Count lines\") +* FUNC is a function which receives a message plist as an argument. + +The first letter of NAME is used as a shortcut character." + :group 'mu4e-view + :type '(alist :key-type string :value-type function)) + +(defcustom mu4e-view-max-specpdl-size 4096 + "The value of `max-specpdl-size' for displaying messages with Gnus." + :type 'integer + :group 'mu4e-view) + + + +(defconst mu4e--view-raw-buffer-name " *mu4e-raw-view*" + "Name for the raw message view buffer.") + +(defun mu4e-view-raw-message () + "Display the raw contents of message at point in a new buffer." + (interactive) + (let ((path (mu4e-message-readable-path)) + (buf (get-buffer-create mu4e--view-raw-buffer-name))) + (with-current-buffer buf + (let ((inhibit-read-only t)) + (erase-buffer) + (mu4e-raw-view-mode) + (insert-file-contents path) + (goto-char (point-min)))) + (mu4e-display-buffer buf t))) + +(defun mu4e-view-pipe (cmd) + "Pipe the message at point through shell command CMD. +Then, display the results." + (interactive "sShell command: ") + (let ((path (mu4e-message-readable-path))) + (mu4e-process-file-through-pipe path cmd))) + +(defmacro mu4e--view-in-headers-context (&rest body) + "Evaluate BODY in the context of the headers buffer." + `(progn + (let* ((msg (mu4e-message-at-point)) + (buffer (cond + ;; are we already inside a headers buffer? + ((mu4e-current-buffer-type-p 'headers) (current-buffer)) + ;; if not, are we inside a view buffer, and does + ;; it have linked headers buffer? + ((mu4e-current-buffer-type-p 'view) + (when (mu4e--view-detached-p (current-buffer)) + (mu4e-error + "Cannot navigate in a detached view buffer.")) + (mu4e-get-headers-buffer)) + ;; fallback; but what would trigger this? + (t (mu4e-get-headers-buffer)))) + (docid (mu4e-message-field msg :docid))) + (unless docid + (mu4e-error "Message without docid: action is not possible")) + + ;; make sure to select the window if possible, or jumping won't be + ;; reflected. + (with-selected-window (or (get-buffer-window buffer) + (get-buffer-window)) + (with-current-buffer buffer + (mu4e-thread-unfold-all) + (if (or (mu4e~headers-goto-docid docid) + ;; TODO: Is this the best way to find another + ;; relevant docid for a view buffer? + ;; + ;; If you attach a view buffer to another headers + ;; buffer that does not contain the current docid + ;; then `mu4e~headers-goto-docid' returns nil and we + ;; get an error. This "hack" instead gets its + ;; now-changed headers buffer's current message as a + ;; docid + (mu4e~headers-goto-docid + (with-current-buffer buffer + (mu4e-message-field (mu4e-message-at-point) :docid)))) + ,@body + (mu4e-error "Cannot find message in headers buffer"))))))) + +(defun mu4e-view-headers-next (&optional n) + "Move point to the next message header. +If this succeeds, return the new docid. Otherwise, return nil. +Optionally, takes an integer N (prefix argument), to the Nth next +header." + (interactive "P") + (mu4e--view-in-headers-context + (mu4e~headers-move (or n 1)))) + +(defun mu4e-view-headers-prev (&optional n) + "Move point to the previous message header. +If this succeeds, return the new docid. Otherwise, return nil. +Optionally, takes an integer N (prefix argument), to the Nth +previous header." + (interactive "P") + (mu4e--view-in-headers-context + (mu4e~headers-move (- (or n 1))))) + +(defun mu4e--view-prev-or-next (func backwards) + "Move point to the next or previous message. +Go to the previous message if BACKWARDS is non-nil. +unread message header in the headers buffer connected with this +message view. If this succeeds, return the new docid. Otherwise, +return nil." + (mu4e--view-in-headers-context (funcall func backwards)) + (mu4e-select-other-view) + (mu4e-headers-view-message)) + +(defun mu4e-view-headers-prev-unread () + "Move point to the previous unread message header. +If this succeeds, return the new docid. Otherwise, return nil." + (interactive) + (mu4e--view-prev-or-next #'mu4e~headers-prev-or-next-unread t)) + +(defun mu4e-view-headers-next-unread () + "Move point to the next unread message header. +If this succeeds, return the new docid. Otherwise, return nil." + (interactive) + (mu4e--view-prev-or-next #'mu4e~headers-prev-or-next-unread nil)) + +(defun mu4e-view-headers-prev-thread() + "Move point to the previous thread. +If this succeeds, return the new docid. Otherwise, return nil." + (interactive) + (mu4e--view-prev-or-next #'mu4e~headers-prev-or-next-thread t)) + +(defun mu4e-view-headers-next-thread() + "Move point to the previous thread. +If this succeeds, return the new docid. Otherwise, return nil." + (interactive) + (mu4e--view-prev-or-next #'mu4e~headers-prev-or-next-thread nil)) + +(defun mu4e-view-thread-goto-root () + "Move to thread root." + (interactive) + (mu4e--view-in-headers-context (mu4e-thread-goto-root))) + +(defun mu4e-view-thread-fold-toggle-goto-next () + "Toggle threading or go to next." + (interactive) + (mu4e--view-in-headers-context (mu4e-thread-fold-toggle-goto-next))) + +(defun mu4e-view-thread-fold-toggle-all () + "Toggle all threads." + (interactive) + (mu4e--view-in-headers-context (mu4e-thread-fold-toggle-all))) + + +;;; Interactive functions +(defun mu4e-view-action (&optional msg) + "Ask user for some action to apply on MSG, then do it. +If MSG is nil apply action to message returned +bymessage-at-point. The actions are specified in +`mu4e-view-actions'." + (interactive) + (let* ((msg (or msg (mu4e-message-at-point))) + (actionfunc (mu4e-read-option "Action: " mu4e-view-actions))) + (funcall actionfunc msg))) + +(defun mu4e-view-mark-pattern () + "Mark messages that match a certain pattern. +Ask user for a kind of mark, (move, delete etc.), a field to +match and a regular expression to match with. Then, mark all +matching messages with that mark." + (interactive) + (mu4e--view-in-headers-context (mu4e-headers-mark-pattern))) + +(defun mu4e-view-mark-thread (&optional markpair) + "Mark whole thread with a certain mark. +Ask user for a kind of mark (move, delete etc.), and apply it +to all messages in the thread at point in the headers view. The +optional MARKPAIR can also be used to provide the mark +selection." + (interactive) + (mu4e--view-in-headers-context + (if markpair (mu4e-headers-mark-thread nil markpair) + (call-interactively 'mu4e-headers-mark-thread)))) + +(defun mu4e-view-mark-subthread (&optional markpair) + "Mark subthread with a certain mark. +Ask user for a kind of mark (move, delete etc.), and apply it +to all messages in the subthread at point in the headers view. +The optional MARKPAIR can also be used to provide the mark +selection." + (interactive) + (mu4e--view-in-headers-context + (if markpair (mu4e-headers-mark-subthread markpair) + (mu4e-headers-mark-subthread)))) + +(defun mu4e-view-search-narrow () + "Run `mu4e-headers-search-narrow' in the headers buffer." + (interactive) + (mu4e--view-in-headers-context (mu4e-search-narrow))) + +(defun mu4e-view-search-edit () + "Run `mu4e-search-edit' in the headers buffer." + (interactive) + (mu4e--view-in-headers-context (mu4e-search-edit))) + +(defun mu4e-mark-region-code () + "Highlight region marked with `message-mark-inserted-region'. +Add this function to `mu4e-view-mode-hook' to enable this feature." + (require 'message) + (let (beg end ov-beg ov-end ov-inv) + (save-excursion + (goto-char (point-min)) + (while (re-search-forward + (concat "^" message-mark-insert-begin) nil t) + (setq ov-beg (match-beginning 0) + ov-end (match-end 0) + ov-inv (make-overlay ov-beg ov-end) + beg ov-end) + (overlay-put ov-inv 'invisible t) + (overlay-put ov-inv 'mu4e-overlay t) + (when (re-search-forward + (concat "^" message-mark-insert-end) nil t) + (setq ov-beg (match-beginning 0) + ov-end (match-end 0) + ov-inv (make-overlay ov-beg ov-end) + end ov-beg) + (overlay-put ov-inv 'invisible t)) + (when (and beg end) + (let ((ov (make-overlay beg end))) + (overlay-put ov 'mu4e-overlay t) + (overlay-put ov 'face 'mu4e-region-code)) + (setq beg nil end nil)))))) + +;;; View Utilities + +(defun mu4e-view-mark-custom () + "Run some custom mark function." + (mu4e--view-in-headers-context + (mu4e-headers-mark-custom))) + +(defun mu4e--view-split-view-p () + "Return t if we're in split-view, nil otherwise." + (member mu4e-split-view '(horizontal vertical))) + + +(defun mu4e-view-detach () + "Detach the view buffer from its headers buffer." + (interactive) + (unless mu4e-linked-headers-buffer + (mu4e-error "This view buffer is already detached.")) + (mu4e-message "Detached view buffer from %s" + (progn mu4e-linked-headers-buffer + (with-current-buffer mu4e-linked-headers-buffer + (when (eq (selected-window) mu4e~headers-view-win) + (setq mu4e~headers-view-win nil))) + (setq mu4e-linked-headers-buffer nil) + ;; automatically rename mu4e-view-article buffer when + ;; detaching; will get renamed back when reattaching + (rename-buffer (make-temp-name (buffer-name)) t)))) + +(defun mu4e-view-attach (headers-buffer) + "Attaches a view buffer to a headers buffer." + (interactive + (list (get-buffer (read-buffer + "Select a headers buffer to attach to: " nil t + (lambda (buf) (with-current-buffer (car buf) + (mu4e-current-buffer-type-p 'headers))))))) + (mu4e-message "Attached view buffer to %s" headers-buffer) + (setq mu4e-linked-headers-buffer headers-buffer) + (with-current-buffer headers-buffer + (setq mu4e~headers-view-win (selected-window)))) + +;;; Scroll commands + +(defun mu4e-view-scroll-up-or-next () + "Scroll-up the current message. +If `mu4e-view-scroll-to-next' is non-nil, and we cannot scroll up +any further, go the next message." + (interactive) + (condition-case nil + (scroll-up) + (error + (when mu4e-view-scroll-to-next + (mu4e-view-headers-next))))) + +(defun mu4e-scroll-up () + "Scroll text of selected window up one line." + (interactive) + (scroll-up 1)) + +(defun mu4e-scroll-down () + "Scroll text of selected window down one line." + (interactive) + (scroll-down 1)) + +;;; Mark commands + +(defun mu4e-view-unmark-all () + "If we're in split-view, unmark all messages. +Otherwise, warn user that unmarking only works in the header +list." + (interactive) + (if (mu4e--view-split-view-p) + (mu4e--view-in-headers-context (mu4e-mark-unmark-all)) + (mu4e-message "Unmarking needs to be done in the header list view"))) + +(defun mu4e-view-unmark () + "If we're in split-view, unmark message at point. +Otherwise, warn user that unmarking only works in the header +list." + (interactive) + (if (mu4e--view-split-view-p) + (mu4e-view-mark-for-unmark) + (mu4e-message "Unmarking needs to be done in the header list view"))) + +(defmacro mu4e--view-defun-mark-for (mark) + "Define a function mu4e-view-mark-for- MARK." + (let ((funcname (intern (format "mu4e-view-mark-for-%s" mark))) + (docstring (format "Mark the current message for %s." mark))) + `(progn + (defun ,funcname () ,docstring + (interactive) + (mu4e--view-in-headers-context + (mu4e-headers-mark-and-next ',mark))) + (put ',funcname 'definition-name ',mark)))) + +(mu4e--view-defun-mark-for move) +(mu4e--view-defun-mark-for refile) +(mu4e--view-defun-mark-for delete) +(mu4e--view-defun-mark-for flag) +(mu4e--view-defun-mark-for unflag) +(mu4e--view-defun-mark-for unmark) +(mu4e--view-defun-mark-for something) +(mu4e--view-defun-mark-for read) +(mu4e--view-defun-mark-for unread) +(mu4e--view-defun-mark-for trash) +(mu4e--view-defun-mark-for untrash) + +(defun mu4e-view-marked-execute () + "Execute the marked actions." + (interactive) + (mu4e--view-in-headers-context + (mu4e-mark-execute-all))) + + +;;; URL handling + +(defvar mu4e--view-link-map nil + "A map of some number->url so we can jump to url by number.") +(put 'mu4e--view-link-map 'permanent-local t) + +(defvar mu4e-view-active-urls-keymap + (let ((map (make-sparse-keymap))) + (define-key map (kbd "<mouse-2>") #'mu4e--view-browse-url-from-binding) + (define-key map (kbd "M-<return>") #'mu4e--view-browse-url-from-binding) + map) + "Keymap used for the URLs inside the body.") + +(defvar mu4e--view-beginning-of-url-regexp + "https?\\://\\|mailto:" + "Regexp that matches the beginning of certain URLs. +Match-string 1 will contain the matched URL, if any.") + + +(defun mu4e--view-browse-url-from-binding (&optional url) + "View in browser the url at point, or click location. +If the optional argument URL is provided, browse that instead. +If the url is mailto link, start writing an email to that address." + (interactive) + (let* (( url (or url (mu4e--view-get-property-from-event 'mu4e-url)))) + (when url + (if (string-match-p "^mailto:" url) + (browse-url-mail url) + (browse-url url))))) + +(defun mu4e--view-get-property-from-event (prop) + "Get the property PROP at point, or the location of the mouse. +The action is chosen based on the `last-command-event'. +Meant to be evoked from interactive commands." + (if (and (eventp last-command-event) + (mouse-event-p last-command-event)) + (let ((posn (event-end last-command-event))) + (when (numberp (posn-point posn)) + (get-text-property + (posn-point posn) + prop + (window-buffer (posn-window posn))))) + (get-text-property (point) prop))) + +;; this is fairly simplistic... +(defun mu4e--view-activate-urls () + "Turn things that look like URLs into clickable things. +Also number them so they can be opened using `mu4e-view-go-to-url'." + (let ((num 0)) + (save-excursion + (setq mu4e--view-link-map ;; buffer local + (make-hash-table :size 32 :weakness nil)) + (goto-char (point-min)) + (while (re-search-forward mu4e--view-beginning-of-url-regexp nil t) + (let ((bounds (thing-at-point-bounds-of-url-at-point))) + (when bounds + (let* ((url (thing-at-point-url-at-point)) + (ov (make-overlay (car bounds) (cdr bounds)))) + (puthash (cl-incf num) url mu4e--view-link-map) + (add-text-properties + (car bounds) + (cdr bounds) + `(face mu4e-link-face + mouse-face highlight + mu4e-url ,url + keymap ,mu4e-view-active-urls-keymap + help-echo + "[mouse-1] or [M-RET] to open the link")) + (overlay-put ov 'mu4e-overlay t) + (overlay-put ov 'after-string + (propertize (format "\u200B[%d]" num) + 'face 'mu4e-url-number-face))))))))) + + +(defun mu4e--view-get-urls-num (prompt &optional multi) + "Ask the user with PROMPT for an URL number for MSG. +The number is [1..n] for URLs \[0..(n-1)] in the message. If +MULTI is nil, return the number for the URL; otherwise (MULTI is +non-nil), accept ranges of URL numbers, as per +`mu4e-split-ranges-to-numbers', and return the corresponding +string." + (let* ((count (hash-table-count mu4e--view-link-map)) (def)) + (when (zerop count) (mu4e-error "No links for this message")) + (if (not multi) + (if (= count 1) + (read-number (mu4e-format "%s: " prompt) 1) + (read-number (mu4e-format "%s (1-%d): " prompt count))) + (progn + (setq def (if (= count 1) "1" (format "1-%d" count))) + (read-string (mu4e-format "%s (default %s): " prompt def) + nil nil def))))) + +(defun mu4e-view-go-to-url (&optional multi) + "Offer to go visit one or more URLs. +If MULTI (prefix-argument) is non-nil, offer to go to a range of URLs." + (interactive "P") + (mu4e--view-handle-urls "URL to visit" + multi + (lambda (url) (mu4e--view-browse-url-from-binding url)))) + +(defun mu4e-view-save-url (&optional multi) + "Offer to save URLs to the kill ring. +If MULTI (prefix-argument) is nil, save a single one, otherwise, offer +to save a range of URLs." + (interactive "P") + (mu4e--view-handle-urls "URL to save" multi + (lambda (url) + (kill-new url) + (mu4e-message "Saved %s to the kill-ring" url)))) + +(defun mu4e-view-fetch-url (&optional multi) + "Offer to fetch (download) URLs. +If MULTI (prefix-argument) is nil, +download a single one, otherwise, offer to fetch a range of +URLs. The urls are fetched to `mu4e-attachment-dir'." + (interactive "P") + (mu4e--view-handle-urls + "URL to fetch" multi + (lambda (url) + (let ((target (concat (mu4e-determine-attachment-dir url) "/" + (file-name-nondirectory url)))) + (url-copy-file url target) + (mu4e-message "Fetched %s -> %s" url target))))) + +(defun mu4e--view-handle-urls (prompt multi urlfunc) + "Handle URLs. +If MULTI is nil, apply URLFUNC to a single uri, otherwise, apply +it to a range of uris. PROMPT is the query to present to the user." + (if multi + (mu4e--view-handle-multi-urls prompt urlfunc) + (mu4e--view-handle-single-url prompt urlfunc))) + +(defun mu4e--view-handle-single-url (prompt urlfunc &optional num) + "Apply URLFUNC to some URL with NUM in the current message. +Prompting the user with PROMPT for the number." + (let* ((num (or num (mu4e--view-get-urls-num prompt))) + (url (gethash num mu4e--view-link-map))) + (unless url (mu4e-warn "Invalid number for URL")) + (funcall urlfunc url))) + +(defun mu4e--view-handle-multi-urls (prompt urlfunc) + "Apply URLFUNC to a a range of URLs in the current message. + +Prompting the user with PROMPT for the numbers. + +Default is to apply it to all URLs, [1..n], where n is the number +of urls. You can type multiple values separated by space, e.g. 1 +3-6 8 will visit urls 1,3,4,5,6 and 8. + +Furthermore, there is a shortcut \"a\" which means all urls, but as +this is the default, you may not need it." + (let* ((linkstr (mu4e--view-get-urls-num + "URL number range (or 'a' for 'all')" t)) + (count (hash-table-count mu4e--view-link-map)) + (linknums (mu4e-split-ranges-to-numbers linkstr count))) + (dolist (num linknums) + (mu4e--view-handle-single-url prompt urlfunc num)))) + +(defun mu4e-view-for-each-uri (func) + "Evaluate FUNC(uri) for each uri in the current message." + (maphash (lambda (_num uri) (funcall func uri)) mu4e--view-link-map)) + +(defun mu4e-view-message-with-message-id (msgid) + "View message with message-id MSGID. +This (re)creates a +headers-buffer with a search for MSGID, then open a view for that +message." + (mu4e-search (concat "msgid:" msgid) nil nil t msgid t)) + +;;; Variables + +(defvar gnus-icalendar-additional-identities) +(defvar-local mu4e--view-rendering nil) + +(defun mu4e-view (msg) + "Display the message MSG in a new buffer, and keep in sync with HDRSBUF. +\"In sync\" here means that moving to the next/previous message +in the the message view affects HDRSBUF, as does marking etc. + +As a side-effect, a message that is being viewed loses its +`unread' marking if it still had that." + ;; update headers, if necessary. + (mu4e~headers-update-handler msg nil nil) + ;; Create a new view buffer (if needed) as it is not + ;; feasible to recycle an existing buffer due to buffer-specific + ;; state (buttons, etc.) that can interfere with message rendering + ;; in gnus. + ;; + ;; Unfortunately that does create its own issues: namely ensuring + ;; buffer-local state that *must* survive is correctly copied + ;; across. + (let ((linked-headers-buffer)) + (when-let ((existing-buffer (mu4e-get-view-buffer nil nil))) + ;; required; this state must carry over from the killed buffer + ;; to the new one. + (setq linked-headers-buffer mu4e-linked-headers-buffer) + (if (memq mu4e-split-view '(horizontal vertical)) + (delete-windows-on existing-buffer t)) + (kill-buffer existing-buffer)) + (setq gnus-article-buffer (mu4e-get-view-buffer nil t)) + (with-current-buffer gnus-article-buffer + (when linked-headers-buffer + (setq mu4e-linked-headers-buffer linked-headers-buffer)) + (let ((inhibit-read-only t) + (gnus-unbuttonized-mime-types '(".*/.*")) + (gnus-buttonized-mime-types + (append (list "multipart/signed" "multipart/encrypted") + gnus-buttonized-mime-types)) + (gnus-inhibit-mime-unbuttonizing t)) + (remove-overlays (point-min)(point-max) 'mu4e-overlay t) + (erase-buffer) + (insert-file-contents-literally + (mu4e-message-readable-path msg) nil nil nil t) + ;; some messages have ^M which causes various rendering + ;; problems later (#2260, #2508), so let's remove those + (article-remove-cr) + (setq-local mu4e--view-message msg) + (mu4e--view-render-buffer msg)) + (mu4e-loading-mode 0))) + (unless (mu4e--view-detached-p gnus-article-buffer) + (with-current-buffer mu4e-linked-headers-buffer + ;; We need this here as we want to avoid displaying the buffer until + ;; the last possible moment --- after the message is rendered in the + ;; view buffer. + ;; + ;; Otherwise, `mu4e-display-buffer' may adjust the view buffer's + ;; window height based on a buffer that has no text in it yet! + (setq-local mu4e~headers-view-win + (mu4e-display-buffer gnus-article-buffer nil)) + (unless (window-live-p mu4e~headers-view-win) + (mu4e-error "Cannot get a message view")) + (select-window mu4e~headers-view-win))) + (with-current-buffer gnus-article-buffer + (let ((inhibit-read-only t)) + (run-hooks 'mu4e-view-rendered-hook)) + ;; only needed on some setups; #2683 + (goto-char (point-min)))) + +(defun mu4e-view-message-text (msg) + "Return the rendered MSG as a string." + (with-temp-buffer + (insert-file-contents-literally + (mu4e-message-readable-path msg) nil nil nil t) + (let ((gnus-inhibit-mime-unbuttonizing nil) + (gnus-unbuttonized-mime-types '(".*/.*")) + (mu4e-view-fields '(:from :to :cc :subject :date))) + (mu4e--view-render-buffer msg) + (buffer-substring-no-properties (point-min) (point-max))))) + +(defun mu4e-action-view-in-browser (msg &optional skip-headers) + "Show current MSG in browser if it includes an HTML-part. +If SKIP-HEADERS is set, do not show include message headers. +The variables `browse-url-browser-function', +`browse-url-handlers', and `browse-url-default-handlers' +determine which browser function to use." + (with-temp-buffer + (insert-file-contents-literally + (mu4e-message-readable-path msg) nil nil nil t) + ;; just continue if some of the decoding fails. + (ignore-errors (run-hooks 'gnus-article-decode-hook)) + (let ((header (unless skip-headers + (cl-loop for field in '("from" "to" "cc" "date" "subject") + when (message-field-value field) + concat (format "%s: %s\n" (capitalize field) it)))) + (parts (mm-dissect-buffer t t))) + ;; If singlepart, enforce a list. + (when (and (bufferp (car parts)) + (stringp (car (mm-handle-type parts)))) + (setq parts (list parts))) + ;; Process the list + (unless (gnus-article-browse-html-parts parts header) + (mu4e-warn "Message does not contain a \"text/html\" part")) + (mm-destroy-parts parts)))) + +(defun mu4e-action-view-in-xwidget (msg) + "Show current MSG in an embedded xwidget, if available." + (unless (fboundp 'xwidget-webkit-browse-url) + (mu4e-error "No xwidget support available")) + (let ((browse-url-handlers nil) + (browse-url-browser-function + (lambda (url &optional _rest) + (xwidget-webkit-browse-url url)))) + (mu4e-action-view-in-browser msg))) + +(defun mu4e--view-render-buffer (msg) + "Render current buffer with MSG using Gnus' article mode." + (setq gnus-summary-buffer (get-buffer-create " *appease-gnus*")) + (let* ((inhibit-read-only t) + (max-specpdl-size mu4e-view-max-specpdl-size) + (mm-decrypt-option 'known) + (ct (mail-fetch-field "Content-Type")) + (ct (and ct (mail-header-parse-content-type ct))) + (charset (mail-content-type-get ct 'charset)) + (charset (and charset (intern charset))) + (mu4e--view-rendering t); Needed if e.g. an ics file is buttonized + (gnus-article-emulate-mime nil) ;; avoid perf problems + (gnus-newsgroup-charset + (if (and charset (coding-system-p charset)) charset + (detect-coding-region (point-min) (point-max) t))) + ;; Possibly add headers (before "Attachments") + (gnus-display-mime-function (mu4e--view-gnus-display-mime msg))) + (condition-case err + (progn + (mm-enable-multibyte) + ;; just continue if some of the decoding fails. + (ignore-errors (run-hooks 'gnus-article-decode-hook)) + (gnus-article-prepare-display) + (mu4e--view-activate-urls) + ;; `gnus-summary-bookmark-make-record' does not work properly when "appeased." + (kill-local-variable 'bookmark-make-record-function) + (setq mu4e~gnus-article-mime-handles gnus-article-mime-handles + gnus-article-decoded-p gnus-article-decode-hook) + (set-buffer-modified-p nil) + (add-hook 'kill-buffer-hook #'mu4e--view-kill-mime-handles)) + (epg-error + (mu4e-warn "EPG error: %s; fall back to raw view" + (error-message-string err)))))) + +(defun mu4e-view-refresh () + "Refresh the message view." + ;;; XXX: sometimes, side-effect: increase the header-buffers size + (interactive) + (when-let ((msg (and (derived-mode-p 'mu4e-view-mode) + mu4e--view-message))) + (mu4e-view-quit) + (mu4e-view msg))) + +(defun mu4e-view-toggle-show-mime-parts() + "Toggle whether to show all MIME-parts." + (interactive) + (setq gnus-inhibit-mime-unbuttonizing + (not gnus-inhibit-mime-unbuttonizing)) + (mu4e-view-refresh)) + +(defun mu4e-view-toggle-fill-flowed() + "Toggle flowed-message text filling." + (interactive) + (setq mm-fill-flowed (not mm-fill-flowed)) + (mu4e-view-refresh)) + +(defun mu4e-view-toggle-emulate-mime() + "Toggle GNUs MIME-emulation. +Note that for some messages, this can trigger high CPU load." + (interactive) + (setq gnus-article-emulate-mime (not gnus-article-emulate-mime)) + (mu4e-view-refresh)) + +(defun mu4e--view-gnus-display-mime (msg) + "Like `gnus-display-mime', but include mu4e headers to MSG." + (lambda (&optional ihandles) + (gnus-display-mime ihandles) + (unless ihandles + (save-restriction + (article-goto-body) + (forward-line -1) + (narrow-to-region (point) (point)) + (dolist (field mu4e-view-fields) + (let ((fieldval (mu4e-message-field msg field))) + (pcase field + ((or ':path ':maildir ':list) + (mu4e--view-gnus-insert-header field fieldval)) + (':message-id + (when-let ((msgid (plist-get msg :message-id))) + (mu4e--view-gnus-insert-header field (format "<%s>" msgid)))) + (':mailing-list + (let ((list (plist-get msg :list))) + (if list (mu4e-get-mailing-list-shortname list) ""))) + ((or ':flags ':tags) + (let ((flags (mapconcat (lambda (flag) + (if (symbolp flag) + (symbol-name flag) + flag)) fieldval ", "))) + (mu4e--view-gnus-insert-header field flags))) + (':size (mu4e--view-gnus-insert-header + field (mu4e-display-size fieldval))) + ((or ':subject ':to ':from ':cc ':bcc ':from-or-to + ':user-agent ':date ':attachments + ':signature ':decryption)) ;; handled by Gnus + (_ + (mu4e--view-gnus-insert-header-custom msg field))))) + (let ((gnus-treatment-function-alist + '((gnus-treat-highlight-headers + gnus-article-highlight-headers)))) + (gnus-treat-article 'head)))))) + +(defun mu4e--view-gnus-insert-header (field val) + "Insert a header FIELD with value VAL." + (let* ((info (cdr (assoc field mu4e-header-info))) + (key (plist-get info :name)) + (help (plist-get info :help))) + (if (and val (> (length val) 0)) + (insert (propertize (concat key ":") 'help-echo help) + " " val "\n")))) + +(defun mu4e--view-gnus-insert-header-custom (msg field) + "Insert MSG's custom FIELD." + (let* ((info (cdr-safe (or (assoc field mu4e-header-info-custom) + (mu4e-error "Custom field %S not found" field)))) + (key (plist-get info :name)) + (func (or (plist-get info :function) + (mu4e-error "No :function defined for custom field %S %S" + field info))) + (val (funcall func msg)) + (help (plist-get info :help))) + (when (and val (> (length val) 0)) + (insert (propertize (concat key ":") 'help-echo help) " " val "\n")))) + +(define-advice gnus-icalendar-event-from-handle + (:filter-args (handle-attendee) mu4e--view-fix-missing-charset) + "Avoid error when displaying an ical attachment without a charset." + (if (and (boundp 'mu4e--view-rendering) mu4e--view-rendering) + (let* ((handle (car handle-attendee)) + (attendee (cadr handle-attendee)) + (buf (mm-handle-buffer handle)) + (ty (mm-handle-type handle)) + (rest (cddr handle))) + ;; Put the fallback at the end: + (setq ty (append ty '((charset . "utf-8")))) + (setq handle (cons buf (cons ty rest))) + (list handle attendee)) + handle-attendee)) + +(defun mu4e--view-mode-p () + "Is the buffer in mu4e-view-mode or one of its descendants?" + (or (eq major-mode 'mu4e-view-mode) + (derived-mode-p '(mu4e-view-mode)))) + +(defun mu4e--view-nop (func &rest args) + "Do not invoke FUNC with ARGS when in mu4e-view-mode. +This is useful for advising some Gnus-functionality that does not work in mu4e." + (unless (mu4e--view-mode-p) + (apply func args))) + +(defun mu4e--view-button-reply (func &rest args) + "Advise FUNC with ARGS to make `gnus-button-reply' links work in mu4e." + (if (mu4e--view-mode-p) + (mu4e-compose-reply) + (apply func args))) + +(defun mu4e--view-button-message-id (func &rest args) + "Advise FUNC with ARGS to make `gnus-button-message-id' links work in mu4e." + (if (and (mu4e--view-mode-p) (stringp (car-safe args))) + (mu4e-view-message-with-message-id (car args)) + (apply func args))) + +(defun mu4e--view-msg-mail (func &rest args) + "Advise FUNC with ARGS to make `gnus-msg-mail' links compose with mu4e." + (if (mu4e--view-mode-p) + (apply 'mu4e-compose-mail args) + (apply func args))) + +(defun mu4e-view-quit () + "Quit the mu4e-view buffer." + (interactive) + (if (memq mu4e-split-view '(horizontal vertical)) + (ignore-errors ;; try, don't error out. + (kill-buffer-and-window)) + ;; single-window case + (let ((docid (mu4e-field-at-point :docid))) + (when mu4e-linked-headers-buffer ;; re-use mu4e-view-detach? + (with-current-buffer mu4e-linked-headers-buffer + (when (eq (selected-window) mu4e~headers-view-win) + (setq mu4e~headers-view-win nil))) + (setq mu4e-linked-headers-buffer nil) + (kill-buffer) + ;; attempt to move point to just-viewed message. + (when docid + (ignore-errors + (mu4e~headers-goto-docid docid))))))) + +(defvar mu4e-view-mode-map + (let ((map (make-keymap))) + (define-key map (kbd "C-S-u") #'mu4e-update-mail-and-index) + (define-key map (kbd "C-c C-u") #'mu4e-update-mail-and-index) + + (define-key map "q" #'mu4e-view-quit) + + (define-key map "z" #'mu4e-view-detach) + (define-key map "Z" #'mu4e-view-attach) + + (define-key map "%" #'mu4e-view-mark-pattern) + (define-key map "t" #'mu4e-view-mark-subthread) + (define-key map "T" #'mu4e-view-mark-thread) + + (define-key map "g" #'mu4e-view-go-to-url) + (define-key map "k" #'mu4e-view-save-url) + (define-key map "f" #'mu4e-view-fetch-url) + + (define-key map "." #'mu4e-view-raw-message) + (define-key map "," #'mu4e-sexp-at-point) + (define-key map "|" #'mu4e-view-pipe) + (define-key map "a" #'mu4e-view-action) + (define-key map "A" #'mu4e-view-mime-part-action) + (define-key map "e" #'mu4e-view-save-attachments) + + ;; change the number of headers + (define-key map (kbd "C-+") #'mu4e-headers-split-view-grow) + (define-key map (kbd "C--") #'mu4e-headers-split-view-shrink) + (define-key map (kbd "<C-kp-add>") #'mu4e-headers-split-view-grow) + (define-key map (kbd "<C-kp-subtract>") #'mu4e-headers-split-view-shrink) + + ;; intra-message navigation + (define-key map (kbd "S-SPC") #'scroll-down) + (define-key map (kbd "SPC") #'mu4e-view-scroll-up-or-next) + (define-key map (kbd "RET") #'mu4e-scroll-up) + (define-key map (kbd "<backspace>") #'mu4e-scroll-down) + + ;; navigation between messages + (define-key map "p" #'mu4e-view-headers-prev) + (define-key map "n" #'mu4e-view-headers-next) + ;; the same + (define-key map (kbd "<M-down>") #'mu4e-view-headers-next) + (define-key map (kbd "<M-up>") #'mu4e-view-headers-prev) + + (define-key map (kbd "[") #'mu4e-view-headers-prev-unread) + (define-key map (kbd "]") #'mu4e-view-headers-next-unread) + (define-key map (kbd "{") #'mu4e-view-headers-prev-thread) + (define-key map (kbd "}") #'mu4e-view-headers-next-thread) + + ;; ;; threads + ;; TODO: find some binding that don't conflict + ;; (define-key map (kbd "<S-left>") #'mu4e-view-thread-goto-root) + ;; ;; <tab> is taken already + ;; (define-key map (kbd "<C-S-tab>") #'mu4e-view-thread-fold-toggle-goto-next) + ;; (define-key map (kbd "<backtab>") #'mu4e-view-thread-fold-toggle-all) + + + ;; switching from view <-> headers (when visible) + (define-key map "y" #'mu4e-select-other-view) + + ;; marking/unmarking + (define-key map "d" #'mu4e-view-mark-for-trash) + (define-key map (kbd "<delete>") #'mu4e-view-mark-for-delete) + (define-key map (kbd "<deletechar>") #'mu4e-view-mark-for-delete) + (define-key map (kbd "D") #'mu4e-view-mark-for-delete) + (define-key map (kbd "m") #'mu4e-view-mark-for-move) + (define-key map (kbd "r") #'mu4e-view-mark-for-refile) + + (define-key map (kbd "?") #'mu4e-view-mark-for-unread) + (define-key map (kbd "!") #'mu4e-view-mark-for-read) + + (define-key map (kbd "+") #'mu4e-view-mark-for-flag) + (define-key map (kbd "-") #'mu4e-view-mark-for-unflag) + (define-key map (kbd "=") #'mu4e-view-mark-for-untrash) + (define-key map (kbd "&") #'mu4e-view-mark-custom) + + (define-key map (kbd "*") #'mu4e-view-mark-for-something) + (define-key map (kbd "<kp-multiply>") #'mu4e-view-mark-for-something) + (define-key map (kbd "<insert>") #'mu4e-view-mark-for-something) + (define-key map (kbd "<insertchar>") #'mu4e-view-mark-for-something) + + (define-key map ";" #'mu4e-context-switch) + + (define-key map (kbd "#") #'mu4e-mark-resolve-deferred-marks) + ;; misc + (define-key map "M" #'mu4e-view-massage) + + (define-key map "w" #'visual-line-mode) + (define-key map "h" #'mu4e-view-toggle-html) + (define-key map (kbd "M-q") #'article-fill-long-lines) + + (define-key map "c" #'mu4e-copy-thing-at-point) + + ;; next 3 only warn user when attempt in the message view + (define-key map "u" #'mu4e-view-unmark) + (define-key map "U" #'mu4e-view-unmark-all) + (define-key map "x" #'mu4e-view-marked-execute) + + (define-key map "$" #'mu4e-show-log) + (define-key map "H" #'mu4e-display-manual) + + ;; Make 0..9 shortcuts for digit-argument. Actually, none of the bound + ;; functions seem to use a prefix arg but those bindings existed because we + ;; used to use `suppress-keymap'. And possibly users added their own + ;; prefix arg consuming commands. + (dotimes (i 10) + (define-key map (kbd (format "%d" i)) #'digit-argument)) + + (set-keymap-parent map special-mode-map) + (set-keymap-parent map button-buffer-map) + map) + "Keymap for mu4e-view mode.") + +(easy-menu-define mu4e-view-mode-menu + mu4e-view-mode-map "Menu for mu4e's view-mode." + (append + '("View" + "--" + ["Toggle wrap lines" visual-line-mode] + ["View raw" mu4e-view-raw-message] + ["Pipe through shell" mu4e-view-pipe] + "--" + ["Mark for deletion" mu4e-view-mark-for-delete] + ["Mark for untrash" mu4e-view-mark-for-untrash] + ["Mark for trash" mu4e-view-mark-for-trash] + ["Mark for move" mu4e-view-mark-for-move] + ) + mu4e--compose-menu-items + mu4e--search-menu-items + mu4e--context-menu-items + '( + "--" + ["Quit" mu4e-view-quit + :help "Quit the view"] + ))) + +(defcustom mu4e-raw-view-mode-hook nil + "Hook run when entering \\[mu4e-raw-view] mode." + :options '() + :type 'hook + :group 'mu4e-view) + +(defcustom mu4e-view-mode-hook nil + "Hook run when entering \\[mu4e-view] mode." + :options '(turn-on-visual-line-mode) + :type 'hook + :group 'mu4e-view) + +(defcustom mu4e-view-rendered-hook '(mu4e-resize-linked-headers-window) + "Hook run by `mu4e-view' after a message is rendered." + :type 'hook + :group 'mu4e-view) + +(define-derived-mode mu4e-raw-view-mode fundamental-mode "mu4e:raw-view" + (view-mode)) + +;; "Define the major-mode for the mu4e-view." +(define-derived-mode mu4e-view-mode gnus-article-mode "mu4e:view" + "Major mode for viewing an e-mail message in mu4e. +Based on Gnus' article-mode." + ;; some external tools (bbdb) depend on this + (setq gnus-article-buffer (current-buffer)) + + ;; ;; turn off gnus modeline changes and menu items + (advice-add 'gnus-set-mode-line :around #'mu4e--view-nop) + (advice-add 'gnus-button-reply :around #'mu4e--view-button-reply) + (advice-add 'gnus-button-message-id :around #'mu4e--view-button-message-id) + (advice-add 'gnus-msg-mail :around #'mu4e--view-msg-mail) + + ;; advice gnus-block-private-groups to always return "." + ;; so that by default we block images. + (advice-add 'gnus-block-private-groups :around + (lambda(func &rest args) + (if (mu4e--view-mode-p) + "." (apply func args)))) + (use-local-map mu4e-view-mode-map) + (mu4e-context-minor-mode) + (mu4e-search-minor-mode) + (mu4e-compose-minor-mode) + (setq buffer-undo-list t) ;; don't record undo info + + ;; support bookmarks. + (set (make-local-variable 'bookmark-make-record-function) + 'mu4e--make-bookmark-record) + + ;; autopair mode gives error when pressing RET + ;; turn it off + (when (boundp 'autopair-dont-activate) + (setq autopair-dont-activate t))) + +;;; Massaging the message view + +(defcustom mu4e-view-massage-options + '( ("ctoggle citations" . gnus-article-hide-citation) + ("htoggle headers" . gnus-article-hide-headers) + ("ytoggle crypto" . gnus-article-hide-pem) + ("ftoggle fill-flowed" . mu4e-view-toggle-fill-flowed) + ("mtoggle show all MIME parts" . mu4e-view-toggle-show-mime-parts) + ("Mtoggle show emulate MIME" . mu4e-view-toggle-emulate-mime)) +"Various options for \"massaging\" the message view. See `(gnus) +Article Treatment' for more options." + :group 'mu4e-view + :type '(alist :key-type string :value-type function)) + +(defun mu4e-view-massage() + "Massage current message view as per `mu4e-view-massage-options'." + (interactive) + (funcall (mu4e-read-option "Massage: " mu4e-view-massage-options))) + + +(defun mu4e-view-toggle-html () + "Toggle html-display of the first html-part found." + (interactive) + ;; This function assumes `gnus-article-mime-handle-alist' is sorted by + ;; pertinence, i.e. the first HTML part found in it is the most important one. + (save-excursion + (if-let ((html-part + (seq-find (lambda (handle) + (equal (mm-handle-media-type (cdr handle)) + "text/html")) + gnus-article-mime-handle-alist)) + (text-part + (seq-find (lambda (handle) + (equal (mm-handle-media-type (cdr handle)) + "text/plain")) + gnus-article-mime-handle-alist))) + (gnus-article-inline-part (car html-part)) + (mu4e-warn "Cannot switch; no html and/or text part in this message")))) + +;;; Bug Reference mode support + +;; Due to mu4e's view buffer handling (mu4e-view-mode is called long before the +;; actual mail text is inserted into the buffer), one should activate +;; bug-reference-mode in mu4e-after-view-message-hook, not mu4e-view-mode-hook. + +;; This is Emacs 28 stuff but there is no need to guard it with some (f)boundp +;; checks (which would return nil if bug-reference.el is not loaded before +;; mu4e) since the function definition doesn't hurt and `add-hook' works fine +;; for not yet defined variables (by creating them). +(declare-function bug-reference-maybe-setup-from-mail "ext:bug-reference") + +(defvar mu4e--view-bug-reference-checked-headers + '("list" "list-id" "to" "from" "cc" "subject" "reply-to") + "List of mail headers whose values are passed to bug-reference's auto-setup.") + +(defun mu4e--view-try-setup-bug-reference-mode () + "Try to guess bug-reference setup from the current mu4e mail. +Looks at the maildir and the mail headers in +`mu4e--view-bug-reference-checked-headers' and tries to guess suitable +values for `bug-reference-bug-regexp' and +`bug-reference-url-format' by matching the maildir name against +GROUP-REGEXP and each header value against HEADER-REGEXP in +`bug-reference-setup-from-mail-alist'." + (when (derived-mode-p 'mu4e-view-mode) + (let (header-values) + (save-excursion + (goto-char (point-min)) + (dolist (field mu4e--view-bug-reference-checked-headers) + (let ((val (mail-fetch-field field))) + (when val + (push val header-values))))) + (bug-reference-maybe-setup-from-mail + (mail-fetch-field "maildir") + header-values)))) + +(with-eval-after-load 'bug-reference + (add-hook 'bug-reference-auto-setup-functions + #'mu4e--view-try-setup-bug-reference-mode)) + + +(provide 'mu4e-view) +;;; mu4e-view.el ends here diff --git a/mu4e/mu4e-window.el b/mu4e/mu4e-window.el new file mode 100644 index 0000000..af2e933 --- /dev/null +++ b/mu4e/mu4e-window.el @@ -0,0 +1,383 @@ +;;; mu4e-window.el --- Window management -*- lexical-binding: t; -*- + +;; Copyright (C) 2022 Mickey Petersen +;; Copyright (C) 2023-2024 Dirk-Jan C. Binnema + +;; Author: Mickey Petersen <mickey@masteringemacs.org> +;; Keywords: mail + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <https://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: + +;;; Buffer names for internal use + +(defconst mu4e--sexp-buffer-name "*mu4e-sexp-at-point*" + "Buffer name for sexp buffers.") + +(defvar mu4e-main-buffer-name "*mu4e-main*" + "Name of the mu4e main buffer.") + +(defvar mu4e-embedded-buffer-name " *mu4e-embedded*" + "Name for the embedded message view buffer.") + +;; Buffer names for public use + +(defvar mu4e-headers-buffer-name "*mu4e-headers*" + "Name of the buffer for message headers.") + +(defvar mu4e-view-buffer-name "*mu4e-article*" + "Name of the view buffer.") + +(defvar mu4e-headers-buffer-name-func nil + "Function used to name the headers buffers.") + +(defvar mu4e-view-buffer-name-func nil + "Function used to name the view buffers. + +The function is given one argument, the headers buffer it is +linked to.") + +(defvar-local mu4e-linked-headers-buffer nil + "Holds the headers buffer object that ties it to a view.") + +(defcustom mu4e-split-view 'horizontal + "How to show messages / headers. +A symbol which is either: + * `horizontal': split horizontally (headers on top) + * `vertical': split vertically (headers on the left). + * `single-window': view and headers in one window (mu4e will try not to + touch your window layout), main view in minibuffer + * anything else: don't split (show either headers or messages, + not both). + +Also see `mu4e-headers-visible-lines' and +`mu4e-headers-visible-columns'. + +Note that in older mu4e version, the value could also be +function; this is no longer supported; instead you can use +`display-buffer-alist'." + :type '(choice (const :tag "Split horizontally" horizontal) + (const :tag "Split vertically" vertical) + (const :tag "Single window" single-window) + (const :tag "Don't split" nil)) + :group 'mu4e-headers) + +(defcustom mu4e-headers-visible-lines 10 + "Number of lines to display in the header view when using the +horizontal split-view. This includes the header-line at the top, +and the mode-line." + :type 'integer + :group 'mu4e-headers) + +(defcustom mu4e-headers-visible-columns 30 + "Number of columns to display for the header view when using the +vertical split-view." + :type 'integer + :group 'mu4e-headers) + +(defcustom mu4e-compose-switch nil + "Where to display the new message? +A symbol: +- nil : default (new buffer) +- window : compose in new window +- frame or t : compose in new frame +- display-buffer: use `display-buffer' / `display-buffer-alist' + (for fine-tuning). + +For backward compatibility with `mu4e-compose-in-new-frame', t is +treated as =\\'frame." + :type 'symbol + :group 'mu4e-compose) + +(declare-function mu4e-view-mode "mu4e-view") +(declare-function mu4e-error "mu4e-helpers") +(declare-function mu4e-warn "mu4e-helpers") +(declare-function mu4e-message "mu4e-helpers") + +(defun mu4e-get-headers-buffer (&optional buffer-name create) + "Return a related headers buffer optionally named BUFFER-NAME. + +If CREATE is non-nil, the headers buffer is created if the +generated name does not already exist." + (let* ((buffer-name + (or + ;; buffer name generator func. If a user wants + ;; to supply its own naming scheme, we use that + ;; in lieu of our own heuristic. + (and mu4e-headers-buffer-name-func + (funcall mu4e-headers-buffer-name-func)) + ;; if we're supplied a buffer name for a + ;; headers buffer then try to use that one. + buffer-name + ;; if we're asking for a headers buffer from a + ;; view, then we get our linked buffer. If + ;; there is no such linked buffer -- it is + ;; detached -- raise an error. + (and (mu4e-current-buffer-type-p 'view) + mu4e-linked-headers-buffer) + ;; if we're already in a headers buffer then + ;; that is the one we use. + (and (mu4e-current-buffer-type-p 'headers) + (current-buffer)) + ;; default name to use if all other checks fail. + mu4e-headers-buffer-name)) + (buffer (get-buffer buffer-name))) + (when (and (not (buffer-live-p buffer)) create) + (setq buffer (get-buffer-create buffer-name))) + ;; This may conceivably return a non-existent buffer if `create' + ;; and `buffer-live-p' are nil. + ;; + ;; This is seemingly "OK" as various parts of the code check for + ;; buffer liveness themselves. + buffer)) + +(defun mu4e-get-view-buffers (pred) + "Filter all known view buffers and keep those where PRED return non-nil. + +The PRED function is called from inside the buffer that is being +tested." + (seq-filter + (lambda (buf) + (with-current-buffer buf + (and (mu4e-current-buffer-type-p 'view) + (and pred (funcall pred buf))))) + (buffer-list))) + +(defun mu4e--view-detached-p (buffer) + "Return non-nil if BUFFER is a detached view buffer." + (with-current-buffer buffer + (unless (mu4e-current-buffer-type-p 'view) + (mu4e-error "Buffer `%s' is not a valid mu4e view buffer" buffer)) + (null mu4e-linked-headers-buffer))) + +(defun mu4e--get-current-buffer-type () + "Return an internal symbol that corresponds to each mu4e major mode." + (cond ((or (derived-mode-p 'mu4e-view-mode) + (derived-mode-p 'mu4e-raw-view-mode)) 'view) + ((derived-mode-p 'mu4e-headers-mode) 'headers) + ((derived-mode-p 'mu4e-compose-mode) 'compose) + ((derived-mode-p 'mu4e-main-mode) 'main) + (t 'unknown))) + +(defun mu4e-current-buffer-type-p (type) + "Return non-nil if the current buffer is a mu4e buffer of TYPE. + +Where TYPE is `view', `headers', `compose', `main' or `unknown'. + +Checks are performed using `derived-mode-p' and the current +buffer's major mode." + (eq (mu4e--get-current-buffer-type) type)) + + +;; backward-compat; buffer-local-boundp was introduced in emacs 28. +(defun mu4e--buffer-local-boundp (symbol buffer) + "Return non-nil if SYMBOL is bound in BUFFER. +Also see `local-variable-p'." + (condition-case nil + (buffer-local-value symbol buffer) + (:success t) + (void-variable nil))) + + +(defun mu4e-get-view-buffer (&optional headers-buffer create) + "Return a view buffer belonging optionally to HEADERS-BUFFER. + +If HEADERS-BUFFER is nil, the most likely (and available) headers +buffer is used. + +Detached view buffers are ignored; that may result in a new view buffer +being created if CREATE is non-nil." + ;; If `headers-buffer' is nil, then the caller does not have a + ;; headers buffer preference. + ;; + ;; In that case, we request the most plausible headers buffer from + ;; `mu4e-get-headers-buffer'. + (when (setq headers-buffer (or headers-buffer (mu4e-get-headers-buffer))) + (let ((buffer) + ;; If `mu4e-view-buffer-name-func' is non-nil, then use that + ;; to source the name of the view buffer to create or re-use. + (buffer-name + (or (and mu4e-view-buffer-name-func + (funcall mu4e-view-buffer-name-func headers-buffer)) + ;; If the variable is nil, use the default + ;; name + mu4e-view-buffer-name)) + ;; Search all view buffers and return those that are linked to + ;; `headers-buffer'. + (linked-buffer + (mu4e-get-view-buffers + (lambda (buf) + (and (mu4e--buffer-local-boundp 'mu4e-linked-headers-buffer buf) + (eq mu4e-linked-headers-buffer headers-buffer)))))) + ;; If such a linked buffer exists and its buffer is live, we use that + ;; buffer. + (if (and linked-buffer (buffer-live-p (car linked-buffer))) + ;; NOTE: It's possible for there to be more than one linked view + ;; buffer. + ;; + ;; What, if anything, should the heuristic be to pick the + ;; one to use? Presently `car' is used, but there are better + ;; ways, no doubt. Perhaps preferring those with live windows? + (setq buffer (car linked-buffer)) + (setq buffer (get-buffer buffer-name)) + ;; check if `buffer' is already live *and* detached. If it is, + ;; we'll generate a new, unique name. + (when (and (buffer-live-p buffer) (mu4e--view-detached-p buffer)) + (setq buffer (generate-new-buffer-name buffer-name))) + (when (and (not (buffer-live-p buffer)) create) + (setq buffer (get-buffer-create (or buffer buffer-name))) + (with-current-buffer buffer + (mu4e-view-mode)))) + (when (and buffer (buffer-live-p buffer)) + ;; Required. Callers expect the view buffer to be set. + (set-buffer buffer) + ;; Required. The call chain of `mu4e-view-mode' ends up + ;; calling `kill-all-local-variables', which destroys the + ;; local binding. + (set (make-local-variable 'mu4e-linked-headers-buffer) headers-buffer)) + buffer))) + +;; backward compat: `display-buffer-full-frame' only appears in emacs 29. +(unless (fboundp 'display-buffer-full-frame) + (defun display-buffer-full-frame (buffer alist) + "Display BUFFER in the current frame, taking the entire frame. +ALIST is an association list of action symbols and values. See +Info node `(elisp) Buffer Display Action Alists' for details of +such alists. + +This is an action function for buffer display, see Info +node `(elisp) Buffer Display Action Functions'. It should be +called only by `display-buffer' or a function directly or +indirectly called by the latter." + (when-let ((window (or (display-buffer-reuse-window buffer alist) + (display-buffer-same-window buffer alist) + (display-buffer-pop-up-window buffer alist) + (display-buffer-use-some-window buffer alist)))) + (delete-other-windows window) + window))) + + +(defun mu4e-display-buffer (buffer-or-name &optional select) + "Display BUFFER-OR-NAME as per `mu4e-split-view'. + +If SELECT is non-nil, the final window (and thus BUFFER-OR-NAME) +is selected. + +This function internally uses `display-buffer' (or +`pop-to-buffer' if SELECT is non-nil). + +It is therefore possible to change the display behavior by +modifying `display-buffer-alist'. + +If `mu4e-split-view' is a function, then it must return a live window +for BUFFER-OR-NAME to be displayed in." + ;; For now, using a function for mu4e-split-view is not behaving well + ;; Turn off. + (when (functionp mu4e-split-view) + (mu4e-message "Function for `mu4e-split-view' not supported; fallback") + (setq mu4e-split-view 'horizontal)) + + (let* ((buffer-name (or (get-buffer buffer-or-name) + (mu4e-error "Buffer `%s' does not exist" + buffer-or-name))) + (buffer-type + (with-current-buffer buffer-name (mu4e--get-current-buffer-type))) + (direction (cons 'direction + (pcase (cons buffer-type mu4e-split-view) + ;; views or headers can display + ;; horz/vert depending on the value of + ;; `mu4e-split-view' + (`(,(or 'view 'headers) . horizontal) 'below) + (`(,(or 'view 'headers) . vertical) 'right) + (`(,_ . t) nil)))) + (window-size + (pcase (cons buffer-type mu4e-split-view) + ;; views or headers can display + ;; horz/vert depending on the value of + ;; `mu4e-split-view' + ('(view . horizontal) + '((window-height . shrink-window-if-larger-than-buffer))) + ('(view . vertical) + '((window-min-width . fit-window-to-buffer))) + (`(,_ . t) nil))) + (window-action (cond + ;; main-buffer + ((eq buffer-type 'main) + '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-full-frame)) + ;; compose-buffer + ((eq buffer-type 'compose) + (pcase mu4e-compose-switch + ('window #'display-buffer-pop-up-window) + ((or 'frame 't) #'display-buffer-pop-up-frame) + (_ '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-same-window)))) + ;; headers buffer + ((memq buffer-type '(headers)) + '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-same-window)) + + ((memq mu4e-split-view '(horizontal vertical)) + '(display-buffer-in-direction)) + + ((memq mu4e-split-view '(single-window)) + '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-same-window)) + ;; I cannot discern a difference between + ;; `single-window' and "anything else" in + ;; `mu4e-split-view'. + (t '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-same-window)))) + (arg `((,@window-action) + ,@window-size + ,direction))) + (funcall (if select #'pop-to-buffer #'display-buffer) + buffer-name + arg))) + +(defun mu4e-resize-linked-headers-window () + "Resizes the linked headers window belonging to a view. + +Resizes the current headers view according to `mu4e-split-view' +and `mu4e-headers-visible-lines' or +`mu4e-headers-visible-columns'. + +This function is best called from the hook +`mu4e-view-rendered-hook'." + (unless (mu4e-current-buffer-type-p 'view) + (mu4e-error "Cannot resize as this is not a valid view buffer.")) + (when-let (win (and mu4e-linked-headers-buffer + (get-buffer-window mu4e-linked-headers-buffer))) + ;; This can fail for any number of reasons. If it does, we do + ;; nothing. If the user has customized the window display we may + ;; find it impossible to resize the window, and that should not be + ;; cause for error. + (ignore-errors + (cond ((eq mu4e-split-view 'vertical) + (window-resize win (- mu4e-headers-visible-columns + (window-width win nil)) + t t nil)) + ((eq mu4e-split-view 'horizontal) + (set-window-text-height win mu4e-headers-visible-lines)))))) + +(provide 'mu4e-window) +;;; mu4e-window.el ends here diff --git a/mu4e/mu4e.el b/mu4e/mu4e.el new file mode 100644 index 0000000..d202a3c --- /dev/null +++ b/mu4e/mu4e.el @@ -0,0 +1,266 @@ +;;; mu4e.el --- Mu4e, the mu mail user agent -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Keywords: email + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: +(require 'mu4e-obsolete) + +(require 'mu4e-vars) +(require 'mu4e-window) +(require 'mu4e-helpers) +(require 'mu4e-folders) +(require 'mu4e-context) +(require 'mu4e-contacts) +(require 'mu4e-headers) +(require 'mu4e-search) +(require 'mu4e-view) +(require 'mu4e-compose) +(require 'mu4e-bookmarks) +(require 'mu4e-update) +(require 'mu4e-main) +(require 'mu4e-notification) +(require 'mu4e-server) ;; communication with backend + + + +(when mu4e-speedbar-support + (require 'mu4e-speedbar)) ;; support for speedbar +(when mu4e-org-support + (require 'mu4e-org)) ;; support for org-mode links + +;; We can't properly use compose buffers that are revived using +;; desktop-save-mode; so let's turn that off. +(with-eval-after-load 'desktop + (eval '(add-to-list 'desktop-modes-not-to-save 'mu4e-compose-mode))) + + +;;;###autoload +(defun mu4e (&optional background) + "If mu4e is not running yet, start it. +Then, show the main window, unless BACKGROUND (prefix-argument) +is non-nil." + (interactive "P") + (if (not (mu4e-running-p)) + (progn + (mu4e--init-handlers) + (mu4e--start (unless background #'mu4e--main-view))) + ;; mu4e already running; show unless BACKGROUND + (unless background + (if (buffer-live-p (get-buffer mu4e-main-buffer-name)) + (switch-to-buffer mu4e-main-buffer-name) + (mu4e--main-view))))) + +(defun mu4e-quit(&optional bury) + "Quit the mu4e session or bury the buffer. + +If prefix-argument BURY is non-nil, merely bury the buffer. +Otherwise, completely quit mu4e, including automatic updating." + (interactive "P") + (if bury + (bury-buffer) + (if mu4e-confirm-quit + (when (y-or-n-p (mu4e-format "Are you sure you want to quit?")) + (mu4e--stop)) + (mu4e--stop)))) + +;;; Internals + +(defun mu4e--check-requirements () + "Check for the settings required for running mu4e." + (unless (>= emacs-major-version 25) + (mu4e-error "Emacs >= 25.x is required for mu4e")) + (when (mu4e-server-properties) + (unless (string= (mu4e-server-version) mu4e-mu-version) + (mu4e-error "The mu server has version %s, but we need %s" + (mu4e-server-version) mu4e-mu-version))) + (unless (and mu4e-mu-binary (file-executable-p mu4e-mu-binary)) + (mu4e-error "Please set `mu4e-mu-binary' to the full path to the mu + binary")) + (dolist (var '(mu4e-sent-folder mu4e-drafts-folder + mu4e-trash-folder)) + (unless (and (boundp var) (symbol-value var)) + (mu4e-error "Please set %S" var)) + (unless (functionp (symbol-value var)) ;; functions are okay, too + (let* ((dir (symbol-value var)) + (path (mu4e-join-paths (mu4e-root-maildir) dir))) + (unless (string= (substring dir 0 1) "/") + (mu4e-error "%S must start with a '/'" dir)) + (unless (mu4e-create-maildir-maybe path) + (mu4e-error "%s (%S) does not exist" path var)))))) + +;;; Starting / getting mail / updating the index + +(defun mu4e--pong-handler (_data func) + "Handle \"pong\" responses from the mu server. +Invoke FUNC if non-nil." + (let ((doccount (plist-get (mu4e-server-properties) :doccount))) + (mu4e--check-requirements) + (when func (funcall func)) + (when (zerop doccount) + (mu4e-message "Store is empty; try indexing (M-x mu4e-update-index).")) + (when (and mu4e-update-interval (null mu4e--update-timer)) + (setq mu4e--update-timer + (run-at-time 0 mu4e-update-interval + (lambda () (mu4e-update-mail-and-index + mu4e-index-update-in-background))))))) + +(defun mu4e--start (&optional func) + "Start mu4e. +If `mu4e-contexts' have been defined, but we don't have a context +yet, switch to the matching one, or none matches, the first. If +mu4e is already running, invoke FUNC (if non-nil). + +Otherwise, check requirements, then start mu4e. When successful, invoke + FUNC (if non-nil) afterwards." + (unless (mu4e-context-current) + (mu4e--context-autoswitch nil mu4e-context-policy)) + (setq mu4e-pong-func + (lambda (info) (mu4e--pong-handler info func))) + ;; show some notification? + (when mu4e-notification-support + (add-hook 'mu4e-query-items-updated-hook #'mu4e--notification)) + ;; modeline support + (when mu4e-modeline-support + (mu4e--modeline-register #'mu4e--bookmarks-modeline-item 'global) + (mu4e-modeline-mode) + (add-hook 'mu4e-query-items-updated-hook #'mu4e--modeline-update)) + (mu4e-modeline-mode (if mu4e-modeline-support 1 -1)) + ;; redraw main buffer if there is one. + (add-hook 'mu4e-query-items-updated-hook #'mu4e--main-redraw) + (mu4e--query-items-refresh 'reset-baseline) + (mu4e--server-ping) + ;; ask for the maildir-list + (mu4e--server-data 'maildirs) + ;; maybe request the list of contacts, automatically refreshed after + ;; re-indexing + (unless mu4e--contacts-set + (mu4e--request-contacts-maybe))) + +(defun mu4e--stop () + "Stop mu4e." + (when mu4e--update-timer + (cancel-timer mu4e--update-timer) + (setq mu4e--update-timer nil)) + + (setq ;; clear some caches + mu4e-maildir-list nil + mu4e--contacts-set nil + mu4e--contacts-tstamp "0") + + (remove-hook 'mu4e-query-items-updated-hook #'mu4e--main-redraw) + (remove-hook 'mu4e-query-items-updated-hook #'mu4e--modeline-update) + (remove-hook 'mu4e-query-items-updated-hook #'mu4e--notification) + (mu4e-kill-update-mail) + (mu4e-modeline-mode -1) + (mu4e--server-kill) + ;; kill all mu4e buffers + (mapc + (lambda (buf) + ;; the view buffer has the kill-buffer-hook function + ;; mu4e--view-kill-mime-handles which kills the mm-* buffers created by + ;; Gnus' article mode. Those have been returned by `buffer-list' but might + ;; already be deleted in case the view buffer has been killed first. So we + ;; need a `buffer-live-p' check here. + (when (buffer-live-p buf) + (with-current-buffer buf + (when (member major-mode + '(mu4e-headers-mode mu4e-view-mode mu4e-main-mode)) + (kill-buffer))))) + (buffer-list))) + +;;; Handlers +(defun mu4e--default-handler (&rest args) + "Dummy handler function with arbitrary ARGS." + (mu4e-error "Not handled: %s" args)) + +(defun mu4e--error-handler (errcode errmsg) + "Handler function for showing an error with ERRCODE and ERRMSG." + ;; don't use mu4e-error here; it's running in the process filter context + (pcase errcode + ('4 (mu4e-warn "No matches for this search query.")) + ('110 (display-warning 'mu4e errmsg :error)) ;; schema version. + (_ (mu4e-error "Error %d: %s" errcode errmsg)))) + +(defun mu4e--update-status (info) + "Update the status message with INFO." + (setq mu4e-index-update-status + `(:tstamp ,(current-time) + :checked ,(plist-get info :checked) + :updated ,(plist-get info :updated) + :cleaned-up ,(plist-get info :cleaned-up)))) + +(defun mu4e--info-handler (info) + "Handler function for (:INFO ...) sexps received from server." + (let* ((type (plist-get info :info)) + (checked (plist-get info :checked)) + (updated (plist-get info :updated)) + (cleaned-up (plist-get info :cleaned-up))) + (cond + ((eq type 'add) t) ;; do nothing + ((eq type 'index) + (if (eq (plist-get info :status) 'running) + (mu4e-index-message + "Indexing... checked %d, updated %d" checked updated) + (progn ;; i.e. 'complete + (mu4e--update-status info) + (mu4e-index-message + "%s completed; checked %d, updated %d, cleaned-up %d" + (if mu4e-index-lazy-check "Lazy indexing" "Indexing") + checked updated cleaned-up) + ;; index done; grab updated queries + (mu4e--query-items-refresh) + (run-hooks 'mu4e-index-updated-hook) + ;; backward compatibility... + (unless (zerop (+ updated cleaned-up)) + mu4e-message-changed-hook) + (unless (and (not (string= mu4e--contacts-tstamp "0")) + (zerop (plist-get info :updated))) + (mu4e--request-contacts-maybe) + (mu4e--server-data 'maildirs)) ;; update maildir list + (mu4e--main-redraw)))) + ((plist-get info :message) + (mu4e-index-message "%s" (plist-get info :message)))))) + +(defun mu4e--init-handlers() + "Initialize the server message handlers. +Only set set them if they were nil before, so overriding has a +chance." + (mu4e-setq-if-nil mu4e-error-func #'mu4e--error-handler) + (mu4e-setq-if-nil mu4e-update-func #'mu4e~headers-update-handler) + (mu4e-setq-if-nil mu4e-remove-func #'mu4e~headers-remove-handler) + (mu4e-setq-if-nil mu4e-view-func #'mu4e~headers-view-handler) + (mu4e-setq-if-nil mu4e-headers-append-func #'mu4e~headers-append-handler) + (mu4e-setq-if-nil mu4e-found-func #'mu4e~headers-found-handler) + (mu4e-setq-if-nil mu4e-erase-func #'mu4e~headers-clear) + + (mu4e-setq-if-nil mu4e-sent-func #'mu4e--default-handler) + (mu4e-setq-if-nil mu4e-contacts-func #'mu4e--update-contacts) + (mu4e-setq-if-nil mu4e-info-func #'mu4e--info-handler) + (mu4e-setq-if-nil mu4e-pong-func #'mu4e--default-handler) + + (mu4e-setq-if-nil mu4e-queries-func #'mu4e--query-items-queries-handler)) + +;;; +(provide 'mu4e) +;;; mu4e.el ends here diff --git a/mu4e/mu4e.texi b/mu4e/mu4e.texi new file mode 100644 index 0000000..e3e226c --- /dev/null +++ b/mu4e/mu4e.texi @@ -0,0 +1,4948 @@ +\input texinfo.tex @c -*-texinfo-*- +@documentencoding UTF-8 +@include version.texi +@c %**start of header +@setfilename mu4e.info +@settitle Mu4e @value{VERSION} user manual + +@c Use proper quote and backtick for code sections in PDF output +@c Cf. Texinfo manual 14.2 +@set txicodequoteundirected +@set txicodequotebacktick +@c %**end of header + +@copying +Copyright @copyright{} 2012-@value{UPDATED-YEAR} Dirk-Jan C. Binnema + +@quotation +Permission is granted to copy, distribute and/or modify this document +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A +copy of the license is included in the section entitled ``GNU Free +Documentation License.'' +@end quotation +@end copying + +@titlepage +@title @t{Mu4e} --- an e-mail client for GNU Emacs +@subtitle version @value{VERSION}, @value{UPDATED} +@author Dirk-Jan C. Binnema + +@c The following two commands start the copyright page. +@page +@vskip 0pt plus 1filll +@insertcopying +@end titlepage + +@dircategory Emacs +@direntry +* Mu4e: (Mu4e). An email client for GNU Emacs. +@end direntry + +@contents + +@ifnottex +@node Top +@top mu4e manual for version @value{VERSION} +@end ifnottex + +@iftex +@node Welcome to mu4e +@unnumbered Welcome to mu4e +@end iftex + +Welcome to @t{mu4e}! + +@t{mu4e} (@t{mu}-for-emacs) is an e-mail client for GNU Emacs version 26.3 or +newer, built on top of the @uref{https://www.djcbsoftware.nl/code/mu,mu} e-mail +search engine. @t{mu4e} is optimized for quickly processing large amounts of +e-mail. + +Some of its highlights: +@itemize +@item Fully search-based: there are no folders@footnote{that is, instead of +folders, you use queries that match messages in a particular folder}, +only queries. +@item Fully documented, with example configurations +@item User-interface optimized for speed, with quick key strokes for common actions +@item Support for non-English languages (so ``angstrom'' matches ``Ångström'') +@item Asynchronous: heavy actions don't block @t{emacs}@footnote{currently, +the only exception to this is @emph{sending mail}; there are solutions +for that though --- see the @ref{FAQ}} +@item Support for cryptography --- signing, encrypting and decrypting +@item Address auto-completion based on the contacts in your messages +@item Extendable with your own snippets of elisp +@end itemize + +In this manual, we go through the installation of @t{mu4e}, do some +basic configuration and explain its daily use. We also show you how you +can customize @t{mu4e} for your special needs. + +At the end of the manual, there are some example configurations, to get +you up to speed quickly: @ref{Example configurations}. There's also a section +with answers to frequently asked questions, @ref{FAQ}. + +@menu +* Introduction:: Where to begin +* Getting started:: Setting things up +* Main view:: The @t{mu4e} overview +* Headers view:: Lists of message headers +* Message view:: Viewing specific messages +* Composer:: Creating and editing messages +* Searching:: Some more background on searching/queries` +* Marking:: Marking messages and performing actions +* Contexts:: Defining contexts and switching between them +* Dynamic folders:: Folders that change based on circumstances +* Actions:: Defining and using custom actions +* Extending mu4e:: Writing code for @t{mu4e} +* Integration:: Integrating @t{mu4e} with Emacs facilities + +Appendices +* Other tools:: mu4e and the rest of the world +* Example configurations:: Some examples to set you up quickly +* FAQ:: Common questions and answers +* Tips and Tricks:: Useful tips +* How it works:: Some notes about the implementation of @t{mu4e} +* Debugging:: How to debug problems in @t{mu4e} +* GNU Free Documentation License:: The license of this manual + +Indices +@c * Command Index:: An item for each standard command name. +@c * Variable Index:: An item for each variable documented in this manual. +* Concept Index:: Index of @t{mu4e} concepts and other general subjects. + +@end menu + +@node Introduction +@chapter Introduction + +Let's get started +@menu +* Why another e-mail client::Aren't there enough already +* Other mail clients::Where @t{mu4e} takes its inspiration from +* What mu4e does not do::Focus on the core-business, delegate the rest +* Becoming a mu4e user::Joining the club +@end menu + +@node Why another e-mail client +@section Why another e-mail client? + +I (@t{mu4e}'s author) spend a @emph{lot} of time dealing with e-mail, +both professionally and privately. Having an efficient e-mail client +is essential. Since none of the existing ones worked the way I wanted, +I thought about creating my own. + +Emacs is an integral part of my workflow, so it made a lot of +sense to use it for e-mail as well. And as I had already written an +e-mail search engine (@t{mu}), it seemed only logical to use that as a +basis. + +@node Other mail clients +@section Other mail clients + +Under the hood, @t{mu4e} is fully search-based, similar to programs like +@uref{https://notmuchmail.org/,notmuch} and +@uref{https://sup-heliotrope.github.io/,sup}. + +However, @t{mu4e}'s user-interface is quite different. @t{mu4e}'s mail handling +(deleting, moving, etc.)@: is inspired by +@uref{http://www.gohome.org/wl/,Wanderlust} (another Emacs-based e-mail client), +@uref{http://www.mutt.org/,mutt} and the @t{dired} file-manager for emacs. + +@t{mu4e} keeps all the `state' in your maildirs, so you can easily +switch between clients, synchronize over @abbr{IMAP}, backup with +@t{rsync} and so on. The Xapian-database that @t{mu} maintains is +merely a @emph{cache}; if you delete it, you won't lose any +information. + +@node What mu4e does not do +@section What @t{mu4e} does not do + +There are a number of things that @t{mu4e} does @b{not} do, by design: +@itemize +@item @t{mu}/@t{mu4e} do @emph{not} get your e-mail messages from +a mail server. Nor does it sync-back any changes. Those tasks are delegated to +other tools, such as @uref{https://www.offlineimap.org/,offlineimap}, +@uref{http://isync.sourceforge.net/,mbsync} or +@uref{http://www.fetchmail.info/,fetchmail}; As long as the messages end up in a +maildir, @t{mu4e} and @t{mu} are happy to deal with them. +@item @t{mu4e} also does @emph{not} implement sending of messages; instead, it depends on +@ref{(smtpmail) Top}, which is part of Emacs. In addition, @t{mu4e} piggybacks on +Gnus' message editor. +@end itemize + +Thus, many of the things an e-mail client traditionally needs to do, are +delegated to other tools. This leaves @t{mu4e} to concentrate on what it does +best: quickly finding the mails you are looking for, and handle them as +efficiently as possible. + +@node Becoming a mu4e user +@section Becoming a @t{mu4e} user + +If @t{mu4e} sounds like something for you, give it a shot! We're trying +hard to make it as easy as possible to set up and use; and while you can +use elisp in various places to augment @t{mu4e}, a lot of knowledge +about programming or elisp shouldn't be required. The idea is to provide +sensible defaults, and allow for customization. + +When you take @t{mu4e} into use, it's a good idea to subscribe to the +@uref{https://groups.google.com/group/mu-discuss,mu/mu4e mailing list}. + +Sometimes, you might encounter some unexpected behavior while using @t{mu4e}, or +have some idea on how it could work better. To report this, you can use the +@uref{https://github.com/djcb/mu/issues,bug-tracker}. Please always include the +following information: + +@itemize +@item what did you expect or wish to happen? what actually happened? +@item can you provide some exact steps to reproduce? +@item what version of @t{mu4e} and @t{emacs} were you using? What operating system? +@item can you reproduce it with @command{emacs -q} and only loading @t{mu4e}? +@item if the problem is related to some specific message, please include the raw message file (appropriately anonymized, of course) +@end itemize + +@node Getting started +@chapter Getting started + +In this chapter, we go through the installation of @t{mu4e} and its basic setup. +After we have succeeded in @ref{Getting mail}, and @pxref{Indexing your +messages}, we discuss the @ref{Basic configuration}. + +After these steps, @t{mu4e} should be ready to go! + +@menu +* Requirements:: What is needed +* Versions:: Available stable and development versions +* Installation:: How to install @t{mu} and @t{mu4e} +* Getting mail:: Getting mail from a server +* Initializing the message store:: Settings things up +* Indexing your messages:: Creating and maintaining the index +* Basic configuration:: Settings for @t{mu4e} +* Folders:: Setting up standard folders +* Retrieval and indexing:: Doing it from @t{mu4e} +* Sending mail:: How to send mail +* Running mu4e:: Overview of the @t{mu4e} views + +@end menu + +@node Requirements +@section Requirements + +@t{mu}/@t{mu4e} are known to work on a wide variety of Unix- and Unix-like +systems, including many Linux distributions, OS X and FreeBSD. Emacs 26.3 or +higher is required, as well as @uref{https://xapian.org/,Xapian} and +@uref{http://spruce.sourceforge.net/gmime/,GMime}. + +@t{mu} has optional support for the Guile (Scheme) programming language (version +3.0 or higher). There are also some GUI-toys, which require GTK+ 3.x and Webkit. + +If you intend to compile @t{mu} yourself, you need to have the typical +development tools, such as C and C++17 compilers (both @command{gcc} and +@command{clang} work), @command{meson} and @command{make}, and the development +packages for GMime 3.x, GLib and Xapian. Optionally, you also need the +development packages for GTK+, Webkit and Guile. + +@node Versions +@section Versions + +The stable (release) versions have even minor version numbers, while the +development versions have odd ones. So, for example, 1.10.5 is a stable version, +while the 1.11.9 is the development version. + +The stable versions only receive bug fixes after being released, while the +development versions get new features, fixes, and, perhaps, bugs, and are meant +for people with a tolerance for that. + +There is support for one release branch; so, when the 1.10 release is available +(and a new 1.11 development series start), no more changes are expected for the +1.8 releases. + +@node Installation +@section Installation + +@t{mu4e} is part of @t{mu} --- by installing the latter, the former is +installed as well. Some Linux distributions provide packaged versions of +@t{mu}/@t{mu4e}; if you can use those, there is no need to compile +anything yourself. However, if there are no packages for your +distribution, if they are outdated, or if you want to use the latest +development versions, you can follow the steps below. + +@subsection Dependencies + +The first step is to get some build dependencies. The details depend a +bit on your system's setup / distribution. +@itemize +@item On Debian/Ubuntu and derivatives: +@example +$ sudo apt-get install git meson libgmime-3.0-dev libxapian-dev emacs +@end example +@item On Fedora and related: +@example +$ sudo dnf install git meson gmime30-devel xapian-core-devel emacs +@end example +@item Otherwise, install the equivalent of the above on your system +@end itemize + + +@subsection Getting mu + +The next step is to get the @t{mu} sources. There are two alternatives: +@itemize +@item @emph{Use a stable release} -- download a release from +@url{https://github.com/djcb/mu/releases} +@item @emph{Use an experimental development version} -- get it from the repository, +and @t{git clone https://github.com/djcb/mu.git} +@end itemize + +@subsection Building mu + +What all that in place, let's build and install @t{mu} and @t{mu4e}. +Enter the directory where you unpacked or cloned @t{mu}. Then: + +@example +$ ./configure && make +$ sudo make install +@end example + +Note: if you are familiar with @t{meson}, you can of course use its +commands directly; the @t{make} commands are just a thin wrapper around +that. + +@subsection Installation + +After this, @t{mu} and @t{mu4e} should be installed @footnote{there's a +hard dependency between versions of @t{mu4e} and @t{mu} --- you cannot +combine different versions} on your system, and be available from the +command line and in Emacs. + +You may need to restart Emacs, so it can find @t{mu4e} in its +@code{load-path}. If, even after restarting, Emacs cannot find @t{mu4e}, +you may need to add it to your @code{load-path} explicitly; check where +@t{mu4e} is installed, and add something like the following to your +configuration before trying again: +@lisp +;; the exact path may differ --- check it +(add-to-list 'load-path "/usr/local/share/emacs/site-lisp/mu4e") +@end lisp + +@subsection mu4e and emacs customization + +There is some support for using the Emacs customization system in +@t{mu4e}, but for now, we recommend setting the values manually. Please +refer to @ref{Example configurations} for a couple of examples of this; here we +go through things step-by-step. + +@node Getting mail +@section Getting mail + +In order for @t{mu} (and, by extension, @t{mu4e}) to work, you need to have your +e-mail messages stored in a +@uref{https://en.wikipedia.org/wiki/Maildir, Maildir}; in this manual we use the +term `maildir' for both the standard and the hierarchy of maildirs that store +your messages --- a specific directory structure with one-file-per-message. + +If you are already using a maildir, you are lucky. If not, some setup +is required: +@itemize +@item @emph{Using an external IMAP or POP server} --- if you are using an +@abbr{IMAP} or @abbr{POP} server, you can use tools like @t{getmail}, +@t{fetchmail}, @t{offlineimap} or @t{isync} to download your messages +into a maildir (@file{~/Maildir}, often). Because it is such a common +case, there is a full example of setting @t{mu4e} up with +@t{offlineimap} and Gmail; @pxref{Gmail configuration}. +@item @emph{Using a local mail server} --- if you are using a local mail- server +(such as @t{postfix} or @t{qmail}), you can teach them to deliver into +a maildir as well, maybe in combination with @t{procmail}. A bit of +googling should be able to provide you with the details. +@end itemize + +While a @t{mu} only supports a single Maildir, it can be spread across +different file-systems; and symbolic links are supported. + +@node Initializing the message store +@section Initializing the message store + +The first time you run @t{mu}, you need to initialize its store +(database). The default location for that is @t{~/.cache/mu/xapian}, but +you can change this using the @t{--muhome} option, and remember to pass +that to the other commands as well. Alternatively, you can use an +environment variable @t{MUHOME}. + +Assuming that your maildir is at @file{~/Maildir}, we issue the +following command: +@example + $ mu init --maildir=~/Maildir +@end example + +You can add some e-mail addresses, so @t{mu} recognizes them as yours: + +@example + $ mu init --maildir=~/Maildir --my-address=jim@@example.com \ + --my-address=bob@@example.com +@end example + +@t{mu} remembers the maildir and your addresses and uses them when +indexing messages. If you want to change them, you need to @t{init} +once again. + +The addresses may also be basic PCRE regular expressions, wrapped in +slashes, for example: + +@example + $ mu init --maildir=~/Maildir '--my-address=/foo-.*@@example\.com/' +@end example + +If you want to see the values for your message-store, you can use +@command{mu info}. + +@node Indexing your messages +@section Indexing your messages + +After you have succeeded in @ref{Getting mail} and initialized the +message database, we need to @emph{index} the messages. That is --- we +need to scan the messages in the maildir and store the information +about them in a special database. + +We can do that from @t{mu4e} --- @ref{Main view}, but the first time, +it is a good idea to run it from the command line, which makes it +easier to verify that everything works correctly. + +Assuming that your maildir is at @file{~/Maildir}, we issue the +following command: +@example + $ mu index +@end example + +This should scan your messages and fill the database, and give +progress information while doing so. + +The indexing process may take a few minutes the first time you do it +(for thousands of e-mails); afterwards it is much faster, since @t{mu} +only scans messages that are new or have changed. Indexing is discussed +in full detail in the @t{mu-index} man-page. + +After the indexing process has finished, you can quickly test if +everything worked, by trying some command-line searches, for example +@example + $ mu find hello +@end example + +which lists all messages that match @t{hello}. For more examples of +searches, see @ref{Queries}, or check the @t{mu-find} and @t{mu-easy} +man pages. If all of this worked well, we are well on our way setting +things up; the next step is to do some basic configuration for @t{mu4e}. + +@node Basic configuration +@section Basic configuration + +Before we can start using @t{mu4e}, we need to tell Emacs to load +it. So, add to your @file{~/.emacs} (or its moral equivalent, such as +@file{~/.emacs.d/init.el}) something like: + +@lisp +(require 'mu4e) +@end lisp + +If Emacs complains that it cannot find @t{mu4e}, check your +@code{load-path} and make sure that @t{mu4e}'s installation directory is +part of it. If not, you can add it: + +@lisp +(add-to-list 'load-path MU4E-PATH) +@end lisp + +with @t{MU4E-PATH} replaced with the actual path. + +@node Folders +@section Folders + +The next step is to tell @t{mu4e} where it can find your Maildir, and +some special folders. + +So, for example@footnote{Note that the folders (@t{mu4e-sent-folder}, +@t{mu4e-drafts-folder}, @t{mu4e-trash-folder} and +@t{mu4e-refile-folder}) can also be @emph{functions} that are evaluated +at runtime. This allows for dynamically changing them depending on the +situation. See @ref{Dynamic folders} for details.}: +@lisp +;; these are actually the defaults +(setq + mu4e-sent-folder "/sent" ;; folder for sent messages + mu4e-drafts-folder "/drafts" ;; unfinished messages + mu4e-trash-folder "/trash" ;; trashed messages + mu4e-refile-folder "/archive") ;; saved messages +@end lisp + +The folder (maildir) names are all relative to the root-maildir (see the +output of @command{mu info}). If you use @t{mu4e-context}, see +@ref{Contexts and special folders} for what that means for these special +folders. + +@node Retrieval and indexing +@section Retrieval and indexing with mu4e +@cindex mail retrieval +@cindex indexing +As we have seen, we can do all of the mail retrieval @emph{outside} of +Emacs/@t{mu4e}. However, you can also do it from within +@t{mu4e}. + +@subsection Basics + +To set up mail-retrieval from within @t{mu4e}, set the variable +@code{mu4e-get-mail-command} to the program or shell command you want to +use for retrieving mail. You can then get your e-mail using @kbd{M-x +mu4e-update-mail-and-index}, or @kbd{C-S-u} in all @t{mu4e}-views; +alternatively, you can use @kbd{C-c C-u}, which may be more convenient +if you use emacs in a terminal. + +You can kill the (foreground) update process with @kbd{q}. + +It is possible to update your mail and index periodically in the +background or foreground, by setting the variable +@code{mu4e-update-interval} to the number of seconds between these +updates. If set to @code{nil}, it won't update at all. After you make +changes to @code{mu4e-update-interval}, @t{mu4e} must be restarted +before the changes take effect. By default, this will run in +background and to change it to run in foreground, set +@code{mu4e-index-update-in-background} to @code{nil}. + +After updating has completed, @t{mu4e} keeps the output in a buffer +@t{*mu4e-last-update*}, which you can use for diagnosis if needed. + +@subsection Handling errors during mail retrieval + +If the mail-retrieval process returns with a non-zero exit code, +@t{mu4e} shows a warning (unless @code{mu4e-index-update-error-warning} +is set to @code{nil}), but then try to index your maildirs anyway +(unless @code{mu4e-index-update-error-continue} is set to @code{nil}). + +Reason for these defaults is that some of the mail-retrieval programs +may return non-zero, even when the updating process succeeded; however, +it is hard to tell such pseudo-errors from real ones like `login +failed'. + +If you need more refinement, it may be useful to wrap the mail-retrieval +program in a shell-script, for example @t{fetchmail} returns 1 to +indicate `no mail'; we can handle that with: +@lisp +(setq mu4e-get-mail-command "fetchmail -v || [ $? -eq 1 ]") +@end lisp +A similar approach can be used with other mail retrieval programs, +although not all of them have their exit codes documented. + +@subsection Implicit mail retrieval + +If you don't have a specific command for getting mail, for example +because you are running your own mail-server, you can leave +@code{mu4e-get-mail-command} at @t{"true"} (the default), in which case +@t{mu4e} won't try to get new mail, but still re-index your messages. + +@subsection Speeding up indexing + +If you have a large number of e-mail messages in your store, +(re)indexing might take a while. The defaults for indexing are to +ensure that we always have correct, up-to-date information about your +messages, even if other programs have modified the Maildir. + +The downside of this thoroughness (which is the default) is that it is +relatively slow, something that can be noticeable with large e-mail +corpora on slow file-systems. For a faster approach, you can use the +following: + +@lisp +(setq + mu4e-index-cleanup nil ;; don't do a full cleanup check + mu4e-index-lazy-check t) ;; don't consider up-to-date dirs +@end lisp + +In many cases, the mentioned thoroughness might not be needed, and +these settings give a very significant speed-up. If it does not work +for you (e.g., @t{mu4e} fails to find some new messages), simply leave +at the default. + +Note that you can occasionally run a thorough indexing round using +@code{mu4e-update-index-nonlazy}. + +For further details, please refer to the @t{mu-index} manpage; in +particular, see @t{.noindex} and @t{.noupdate} which can help reducing +the indexing time. + +@subsection Example setup + +A simple setup could look something like: + +@lisp +(setq + mu4e-get-mail-command "offlineimap" ;; or fetchmail, or ... + mu4e-update-interval 300) ;; update every 5 minutes +@end lisp + +A hook @code{mu4e-update-pre-hook} is available which is run right +before starting the process. That can be useful, for example, to +influence, @code{mu4e-get-mail-command} based on the the current +situation (location, time of day, ...). + +It is possible to get notifications when the indexing process does any +updates --- for example when receiving new mail. See +@code{mu4e-index-updated-hook} and some tips on its usage in the +@ref{FAQ}. + +@node Sending mail +@section Sending mail + +@t{mu4e} uses Emacs's @ref{(message) Top,,message-mode} for writing mail. + +For sending mail using @abbr{SMTP}, @t{mu4e} uses @ref{(smtpmail) +Top,,smtpmail}. This package supports many different ways to send mail; please +refer to its documentation for the details. + +Here, we only provide some simple examples --- for more, see @ref{Example +configurations}. + +A very minimal setup: + +@lisp +;; tell message-mode how to send mail +(setq message-send-mail-function 'smtpmail-send-it) +;; if our mail server lives at smtp.example.org; if you have a local +;; mail-server, simply use 'localhost' here. +(setq smtpmail-smtp-server "smtp.example.org") +@end lisp + +Since @t{mu4e} (re)uses the same @t{message mode} and @t{smtpmail} that +Gnus uses, many settings for those also apply to @t{mu4e}. + +@subsection Dealing with sent messages + +By default, @t{mu4e} puts a copy of messages you sent in the folder +determined by @code{mu4e-sent-folder}. In some cases, this may not be +what you want - for example, when using Gmail-over-@abbr{IMAP}, this +interferes with Gmail's handling of the sent messages folder, and you +may end up with duplicate messages. + +You can use the variable @code{mu4e-sent-messages-behavior} to customize +what happens with sent messages. The default is the symbol @code{sent} +which, as mentioned, causes the message to be copied to your +sent-messages folder. Other possible values are the symbols @code{trash} +(the sent message is moved to the trash-folder +(@code{mu4e-trash-folder}), and @code{delete} to simply discard the sent +message altogether (so Gmail can deal with it). + +For Gmail-over-@abbr{IMAP}, you could add the following to your +settings: +@verbatim +;; don't save messages to Sent Messages, Gmail/IMAP takes care of this +(setq mu4e-sent-messages-behavior 'delete) +@end verbatim +And that's it! We should now be ready to go. + +For more complex needs, @code{mu4e-sent-messages-behavior} can also be +a parameter-less function that returns one of the mentioned symbols; +see the built-in documentation for the variable. + +@node Running mu4e +@section Running mu4e + +After following the steps in this chapter, we now (hopefully!) have a +working @t{mu4e} setup. Great! In the next chapters, we walk you +through the various views in @t{mu4e}. + +For your orientation, the diagram below shows how the views relate to each +other, and the default key-bindings to navigate between them. + +@cartouche +@verbatim + + [C] +--------+ [RFCE] + --------> | editor | <-------- + / +--------+ \ + / [RFCE]^ \ +/ | \ ++-------+ [sjbB]+---------+ [RET] +---------+ +| main | <---> | headers | <----> | message | ++-------+ [q] +---------+ [qbBjs] +---------+ + [sjbB] ^ +[.] | [q] + V + +-----+ + | raw | + +-----+ + +Default bindings +---------------- +R: Reply s: search .: raw view (toggle) +F: Forward j: jump-to-maildir q: quit +C: Compose b: bookmark-search +E: Edit B: edit bookmark-search + +@end verbatim +@end cartouche + +@node Main view +@chapter The main view + +After you have installed @t{mu4e} (@pxref{Getting started}), you can start it +with @kbd{M-x mu4e}. @t{mu4e} does some checks to ensure everything is set up +correctly, and then shows you the @t{mu4e} main view. Its major mode is +@code{mu4e-main-mode}. + +@menu +* Overview: MV Overview. What is the main view +* Basic actions::What can we do +* Bookmarks and Maildirs: Bookmarks and Maildirs. Jumping to other places +* Miscellaneous::Notes +@end menu + +@node MV Overview +@section Overview + +The main view looks something like the following: + +@cartouche +@verbatim +* mu4e - mu for emacs version x.y.z + + Basics + + * [j]ump to some maildir + * enter a [s]earch query + * [C]ompose a new message + + Bookmarks + + * [bu] Unread messages 13085(+3)/13085 + * [bt] Today's messages + * [bw] Last 7 days 53(+3)/128 + * [bp] Messages with images 75/2441 + + Maildirs + + * [ja] /archive 2101/18837 + * [ji] /inbox 8(+2)/10 + * [jb] /bulk 33/35 + * [jB] /bulkarchive 179/2090 + * [jm] /mu 694(+1)/17687 + * [jn] /sauron + * [js] /sent + + Misc + + * [;]Switch context + * [U]pdate email & database + * toggle [m]ail sending mode (currently direct) + * [f]lush 1 queued mail + + * [N]ews + * [A]bout mu4e + * [H]elp + * [q]uit + + Info + + * last-updated : Sat Dec 31 16:43:56 2022 + * database-path : /home/pam/.cache/mu/xapian + * maildir : /home/pam/Maildir + * in store : 86179 messages + * personal addresses : /.*example.com/, pam@@example.com +@end verbatim +@end cartouche + +Let's walk through the menu. + +@node Basic actions +@section Basic actions + +First, the @emph{Basics}: +@itemize +@item @t{[j]ump to some maildir}: after pressing @key{j} (``jump''), +@t{mu4e} asks you for a maildir to visit. These are the maildirs you +set in @ref{Basic configuration} and any of your own. If you choose +@key{o} (``other'') or @key{/}, you can choose from all maildirs under +the root-maildir. After choosing a maildir, the messages in that +maildir are listed, in the @ref{Headers view}. +@item @t{enter a [s]earch query}: after pressing @key{s}, @t{mu4e} asks +you for a search query, and after entering one, shows the results in the +@ref{Headers view}. +@item @t{[C]ompose a new message}: after pressing @key{C}, you are dropped in +the @ref{Composer} to write a new message. +@end itemize + +@node Bookmarks and Maildirs +@section Bookmarks and Maildirs + +The next two items in the Main view are @emph{Bookmarks} and @emph{Maildirs}. + +Bookmarks are predefined queries with a descriptive name and a shortcut. In the +example above, we see the default bookmarks. You can pick a bookmark by pressing +@key{b} followed by the specific bookmark's shortcut. If you want to edit the +bookmarked query before invoking it, use @key{B}. + +@cindex baseline +Next to each bookmark are some numbers that indicate the unread(delta)/all +matching messages for the given query, with the delta being the difference in +unread count since some ``baseline'', and only shown when this delta > 0. + +Note that the ``delta'' has its limitations: if you, for instance, deleted 5 +messages and received 5 new one, the ``delta'' would be 0, although there were +changes indeed. So it is mostly useful for tracking changes while you are +@emph{not} using @t{mu4e}. For this reason, you can reset the baseline manually, +e.g. by visiting the main view . + +By comparing current results with the baseline, you can quickly what new +messages have arrived since the last time you looked. + +The baseline@footnote{For debugging, it can be useful to see the time for the +baseline - for that, there is the @code{mu4e-baseline-time} command} . is reset +automatically when switching to the main view, or invoking @code{buffer-revert} +(@kbd{g}) while in the main-view. Visiting the ``favorite'' bookmark does the +same(explained below). + +Bookmarks are stored in the variable @code{mu4e-bookmarks}; you can add +your own and/or replace the default ones; @xref{Bookmarks}. For +instance: +@lisp +(add-to-list 'mu4e-bookmarks + ;; add bookmark for recent messages on the Mu mailing list. + '( :name "Mu7Days" + :key ?m + :query "list:mu-discuss.googlegroups.com AND date:7d..now")) +@end lisp + +There are optional keys @t{:hide} to hide the bookmark from the main menu, but +still have it available (using @key{b})) and @t{:hide-unread} to avoid +generating the unread-number; that can be useful if you have bookmarks for slow +queries. Note that @t{:hide-unread} is implied when the query is not a string; +this for the common case where the query function involves some user input, +which would be disruptive in this case. + +There is also the optional @code{:favorite} property, which at most one bookmark +should have; this bookmark is highlighted in the main view, and its +unread-status is shown in the modeline; @xref{Modeline}, and you can enable +desktop notifications; @xref{Desktop notifications}. We'd recommend creating +such a ``favorite'', which should match message that require your quick +attention: + +@lisp +(add-to-list 'mu4e-bookmarks + ;; bookmark for message that require quick attention + '( :name "Urgent" + :key ?u + :query "maildir:/inbox AND from:boss@@exmaple.com")) +@end lisp + +Note that @t{mu4e} resets the baseline when you are interacting with it (for +instance, when you visit the urgent bookmark, or when you go to the main view); +in such cases, there won't be any further notifications. + +The @emph{Maildirs} item is very similar to Bookmarks -- consider maildirs here +as being a special kind of bookmark query that matches a Maildir. You can +configure this using the variable @code{mu4e-maildir-shortcuts}; see its +docstring and @ref{Maildir searches} for more details. + +@node Miscellaneous +@section Miscellaneous + +Finally, there are some @emph{Misc} (miscellaneous) actions: +@itemize +@item @t{[U]pdate email & database} executes the shell-command in the variable +@code{mu4e-get-mail-command}, and afterwards updates the @t{mu} +database; see @ref{Indexing your messages} and @ref{Getting mail} for +details. +@item @t{[R]eset query-results baseline} this reset the current 'baseline' +for query and updates the screen; see @ref{Bookmarks and Maildirs}. +@item @t{toggle [m]ail sending mode (direct)} toggles between sending +mail directly, and queuing it first (for example, when you are offline), +and @t{[f]lush queued mail} flushes any queued mail. This item is +visible only if you have actually set up mail-queuing. @ref{Queuing +mail} +@item @t{[A]bout mu4e} provides general information about the program +@item @t{[H]elp} shows help information for this view +@item Finally, @t{[q]uit mu4e} quits your @t{mu4e}-session@footnote{@t{mu4e-quit}; or with a @t{C-u} +prefix argument, it merely buries the buffer} +@end itemize + +@node Headers view +@chapter The headers view + +The headers view shows the results of a query. The header-line shows the names +of the fields. Below that, there is a line with those fields, for each matching +message, followed by a footer line. The major-mode for the headers view is +@code{mu4e-headers-mode}. + +@menu +* Overview: HV Overview. What is the Header View +* Keybindings::Do things with your keyboard +* Marking: HV Marking. Selecting messages for doing things +* Sorting and threading::Influencing how headers are shown +* Folding threads:: Showing and hiding thread contents +* Custom headers: HV Custom headers. Adding your own headers +* Actions: HV Actions. Defining and using actions +* Buffer display:: How and where the buffers are displayed +@end menu + +@node HV Overview +@section Overview + +An example headers view: +@cartouche +@verbatim +Date V Flgs From/To List Subject +06:32 Nu To Edmund Dantès GstDev Gstreamer-V4L2SINK ... +15:08 Nu Abbé Busoni GstDev ├> ... +18:20 Nu Pierre Morrel GstDev │└> ... +07:48 Nu To Edmund Dantès GstDev └> ... +2013-03-18 S Jacopo EmacsUsr emacs server on win... +2013-03-18 S Mercédès EmacsUsr └> ... +2013-03-18 S Beachamp EmacsUsr Re: Copying a whole... +22:07 Nu Albert de Moncerf EmacsUsr └> ... +2013-03-18 S Gaspard Caderousse GstDev Issue with GESSimpl... +2013-03-18 Ss Baron Danglars GuileUsr Guile-SDL 0.4.2 ava... +End of search results +@end verbatim +@end cartouche + +Some notes to explain what you see in the example: + +@itemize +@item The fields shown in the headers view can be influenced by customizing +the variable @code{mu4e-headers-fields}; see @code{mu4e-header-info} for +the list of built-in fields. Apart from the built-in fields, you can +also create custom fields using @code{mu4e-header-info-custom}; see +@ref{HV Custom headers} for details. +@item By default, the date is shown with the @t{:human-date} field, which +shows the @emph{time} for today's messages, and the @emph{date} for +older messages. If you do not want to distinguish between `today' and +`older', you can use the @t{:date} field instead. +@item You can customize the date and time formats with the variable +@code{mu4e-headers-date-format} and @code{mu4e-headers-time-format}, +respectively. In the example, we use @code{:human-date}, which shows the +time when the message was sent today, and the date otherwise. +@item By default, the subject is shown using the @t{:subject} field; +however, it is also possible to use @t{:thread-subject}, which shows +the subject of a thread only once, similar to the display of the +@t{mutt} e-mail client. +@item The header field used for sorting is indicated by ``@t{V}'' or +``@t{^}''@footnote{or you can use little graphical triangles; see +variable @code{mu4e-use-fancy-chars}}, corresponding to the sort order +(descending or ascending, respectively). You can influence this by a +mouse click, or @key{O}. Not all fields allow sorting. +@item Instead of showing the @t{From:} and @t{To:} fields separately, you +can use From/To (@t{:from-or-to} in @code{mu4e-headers-fields} as a more +compact way to convey the most important information: it shows @t{From:} +@emph{except} when the e-mail was sent by the user (i.e., you) --- in +that case it shows @t{To:} (prefixed by @t{To}@footnote{You can +customize this by changing the variable +@code{mu4e-headers-from-or-to-prefix} (a cons cell)}, as in the example +above). +@item The `List' field shows the mailing-list a message is sent to; +@code{mu4e} tries to create a convenient shortcut for the mailing-list +name; the variable @code{mu4e-user-mailing-lists} can be used to add +your own shortcuts. You can use @code{mu4e-mailing-list-patterns} to +specify generic shortcuts. For instance, to shorten list names to the +part before @t{-list}, you could use: +@lisp +(setq mu4e-mailing-list-patterns '("\\`\\([-_a-z0-9.]+\\)-list")) +@end lisp +@item The letters in the `Flags' field correspond to the following: D=@emph{draft}, +F=@emph{flagged} (i.e., `starred'), N=@emph{new}, P=@emph{passed} (i.e., +forwarded), R=@emph{replied}, S=@emph{seen}, T=@emph{trashed}, +a=@emph{has-attachment}, x=@emph{encrypted}, s=@emph{signed}, +u=@emph{unread}. The tooltip for this field also contains this information. +@item The subject field also indicates the discussion threads, following +@uref{https://www.jwz.org/doc/threading.html,Jamie Zawinski's mail threading +algorithm}. +@item The headers view is @emph{automatically updated} if any changes are +found during the indexing process, and if there is no current +user-interaction. If you do not want such automatic updates, set +@code{mu4e-headers-auto-update} to @code{nil}. +@item Just before executing a search, a hook-function +@code{mu4e-search-hook} is invoked, which receives the search +expression as its parameter. +@item Also, there is a hook-function @code{mu4e-headers-found-hook} available which +is invoked just after @t{mu4e} has completed showing the messages in the +headers-view. +@end itemize + +@node Keybindings +@section Keybindings + +Using the below key bindings, you can do various things with these +messages; these actions are also listed in the @t{Headers} menu in the +Emacs menu bar. + +@verbatim +key description +=========================================================== +n,p view the next, previous message +],[ move to the next, previous unread message +},{ move to the next, previous thread +y select the message view (if visible) +RET open the message at point in the message view + +searching +--------- +s search +S edit last query +/ narrow the search +b search bookmark +B edit bookmark before search +c search query with completion +j jump to maildir +M-left,\ previous query +M-right next query + +O change sort order +P toggle search property + +marking +------- +d mark for moving to the trash folder += mark for removing trash flag ('untrash') +DEL,D mark for complete deletion +m mark for moving to another maildir folder +r mark for refiling ++,- mark for flagging/unflagging +?,! mark message as unread, read + +u unmark message at point +U unmark *all* messages + +% mark based on a regular expression +T,t mark whole thread, subthread + +<insert>,* mark for 'something' (decide later) +# resolve deferred 'something' marks + +x execute actions for the marked messages + +threads +------- +S-left goto root +TAB toggle threading at current level +S-TAB toggle all threading + +composition +----------- +R,W,F,C reply/reply-to-all/forward/compose +E edit (only allowed for draft messages) + +misc +---- +a execute some custom action on a header +| pipe message through shell command +C-+,C-- increase / decrease the number of headers shown +H get help +C-S-u update mail & reindex +C-c C-u update mail & reindex +q leave the headers buffer +@end verbatim + +Some keybindings are available through minor modes: +@itemize +@item Context; see @pxref{Contexts}. +@item Composition; see @pxref{Composer} and @t{mu4e-compose-minor-mode} +@end itemize + +@node HV Marking +@section Marking + +You can @emph{mark} messages for a certain action, such as deletion or +move. After one or more messages are marked, you can then execute +(@code{mu4e-mark-execute-all}, @key{x}) these actions. This two-step +mark-execute sequence is similar to what e.g. @t{dired} does. It is how +@t{mu4e} tries to be as quick as possible, while avoiding accidents. + +The mark/unmark commands support the @emph{region} (i.e., ``selection'') +--- so, for example, if you select some messages and press @key{DEL}, +all messages in the region are marked for deletion. + +You can mark all messages that match a certain pattern with @key{%}. In +addition, you can mark all messages in the current thread (@key{T}) or +sub-thread (@key{t}). + +When you do a new search or refresh the headers buffer while you still +have marked messages, you are asked what to do with those marks --- +whether to @emph{apply} them before leaving, or @emph{ignore} them. This +behavior can be influenced with the variable +@code{mu4e-headers-leave-behavior}. + +For more information about marking, see @ref{Marking}. + +@node Sorting and threading +@section Sorting and threading + +By default, @t{mu4e} sorts messages by date, in descending order: the +most recent messages are shown at the top. In addition, be default +@t{mu4e} shows the message @emph{threads}, i.e., the tree structure +representing a discussion thread; this also affects the sort order: +the top-level messages are sorted by the date of the @emph{newest} +message in the thread. + +The header field used for sorting is indicated by ``@t{V}'' or +``@t{^}''@footnote{or you can use little graphical triangles; see +variable @code{mu4e-use-fancy-chars}}, indicating the sort order +(descending or ascending, respectively). + +You can change the sort order by clicking the corresponding column with the +mouse, or with @kbd{M-x mu4e-headers-change-sorting} (@key{O}); note that not +all fields can be used for sorting. You can toggle threading on/off through +@kbd{M-x mu4e-headers-toggle-property} or @key{Pt}. For both of these functions, +unless you provide a prefix argument (@key{C-u}), the current search is updated +immediately using the new parameters. You can toggle full-search +(@ref{Searching}) through @kbd{M-x mu4e-headers-toggle-property} as well; or +@key{Pf}. + +Note that with threading enabled, the sorting is exclusively by date, +regardless of the column clicked. + +If you want to change the defaults for these settings, you can use the variables +@code{mu4e-search-sort-field} and @code{mu4e-search-show-threads}, as well as +@code{mu4e-search-change-sorting} to change the sorting of the current search +results. + +@node Folding threads +@section Folding threads + +It is possible to fold threads - that is, visually collapse threads into a +single line (and the reverse), by default using the @key{TAB} and @key{S-TAB} +bindings. Note that the collapsing is always for threads as a whole, not for +sub-threads. + +Folding stops at the @emph{first unread message}, unless you set +@code{mu4e-thread-fold-unread}. Similarly, when a thread has marked messages, +the folding stops at the first marked message. Marking folded messages is not +allowed as it is too error-prone. + +Thread-mode functionality is only available with @code{mu4e-search-threads} +enabled; this triggers a minor mode @code{mu4e-thread-mode} in the headers-view. +For now, this functionality is not available in the message view, due to the +conflicting key bindings. + +If you want to automatically fold all threads after a query, you can use a hook: +@lisp + (add-hook 'mu4e-thread-mode-hook #'mu4e-thread-fold-apply-all) +@end lisp + +By default, single-child threads are @emph{not} collapsed, since it would result +in replacing a single line with the collapsed one. However, if, for consistency, +you also want to fold those, you can use @t{mu4e-thread-fold-single-children}. + +@node HV Custom headers +@section Custom headers + +Sometimes the normal headers that @t{mu4e} offers (Date, From, To, +Subject, etc.)@: may not be enough. For these cases, @t{mu4e} offers +@emph{custom headers} in both the headers-view and the message-view. + +You can do so by adding a description of your custom header to +@code{mu4e-header-info-custom}, which is a list of custom headers. + +Let's look at an example --- suppose we want to add a custom header that +shows the number of recipients for a message, i.e., the sum of the +number of recipients in the @t{To:} and @t{Cc:} fields. Let's further +suppose that our function takes a message-plist as its argument +(@ref{Message functions}). + +@lisp +(add-to-list 'mu4e-header-info-custom + '(:recipnum . + ( :name "Number of recipients" ;; long name, as seen in the message-view + :shortname "Recip#" ;; short name, as seen in the headers view + :help "Number of recipients for this message" ;; tooltip + :function (lambda (msg) + (format "%d" + (+ (length (mu4e-message-field msg :to)) + (length (mu4e-message-field msg :cc)))))))) +@end lisp + +Or, let's get the contents of the Jabber-ID header. + +@lisp +(add-to-list 'mu4e-header-info-custom + '(:jabber-id . + ( :name "Jabber-ID" ;; long name, as seen in the message-view + :shortname "JID" ;; short name, as seen in the headers view + :help "The Jabber ID" ;; tooltip + ;; uses mu4e-fetch-field which is rel. slow, so only appropriate + ;; for mu4e-view-fields, and _not_ mu4e-headers-fields + :function (lambda (msg) + (or (mu4e-fetch-field msg "Jabber-ID") ""))))) +@end lisp + +You can then add the custom header to your @code{mu4e-headers-fields} or +@code{mu4e-view-fields}, just like the built-in headers. However, there is an +important caveat: when your custom header in @code{mu4e-headers-fields}, the +function is invoked for each of your message headers in search results, and if +it is slow, would dramatically slow down @t{mu4e}. + +@node HV Actions +@section Actions + +@code{mu4e-headers-action} (@key{a}) lets you pick custom actions to perform +on the message at point. You can specify these actions using the variable +@code{mu4e-headers-actions}. See @ref{Actions} for the details. + +@t{mu4e} defines some default actions. One of those is for @emph{capturing} a +message: @key{a c} `captures' the current message. Next, when you're editing +some message, you can include the previously captured message as an +attachment, using @code{mu4e-compose-attach-captured-message}. See +@file{mu4e-actions.el} in the @t{mu4e} source distribution for more example +actions. + +@node Buffer display +@section Buffer display + +By default, @t{mu4e} will attempt to manage the display of its own buffers. For +headers and message views, the variable @code{mu4e-split-view} is @t{mu4e's} +built-in way to decide how and where they are shown. + +@subsection Split view +You can control how @t{mu4e} displays its buffers, including the @ref{Headers +view} and the @ref{Message view}, by customizing @code{mu4e-split-view}. There +are several options available: + +@itemize +@item @t{horizontal} (this is the default): display the message view below the +header view. Use @code{mu4e-headers-visible-lines} the set the number of +lines shown (default: 8). +@item @t{vertical}: display the message view on the +right side of the header view. Use @code{mu4e-headers-visible-columns} to set +the number of visible columns (default: 30). +@item @t{single-window}: single window mode. Single-window mode tries to +minimize mu4e window operations (opening, killing, resizing, etc) and buffer +changes, while still retaining the view and headers buffers. In addition, it +replaces @t{mu4e}'s main view with a minibuffer-prompt containing the same +information. +@item anything else: prefer reusing the same window, where possible. +@end itemize + +Note that using a window-returning @emph{function} for @code{mu4e-split-view} is +no longer supported, instead you can use @code{display-buffer-alist}, see +the section on further display customization. + +@noindent +Some useful key bindings in the split view: +@itemize +@item @key{C-+} and @key{C--}: interactively change the number of columns or +headers shown +@item You can change the selected window from the +headers-view to the message-view and vice-versa with +@code{mu4e-select-other-view}, bound to @key{y} +@end itemize + +@subsection Further customization + +However, @t{mu4e}'s display rules are provisional; you can override them +easily by customizing @code{display-buffer-alist}, which governs how Emacs -- +and thus @t{mu4e} -- must display your buffers. + +Let's look at some examples. + +@subsection Fine-tuning the main buffer display + +By default @t{mu4e}'s main buffer occupies the complete frame, but this can be +changed to use the current window: + +@lisp +(add-to-list 'display-buffer-alist + `(,(regexp-quote mu4e-main-buffer-name) + display-buffer-same-window)) +@end lisp + +@subsection Fine-tuning headers buffer display + +You do not need to configure @code{mu4e-split-view} for this to work. In the +absence of explicit rules to the contrary, @t{mu4e} will fall back on the value +you have set in @code{mu4e-split-view}. + +Here is an example that displays the headers buffer in a side window to the +right. It occupies half of the width of the frame. + +@lisp +(add-to-list 'display-buffer-alist + `(,(regexp-quote mu4e-headers-buffer-name) + display-buffer-in-side-window + (side . right) + (window-width . 0.5))) +@end lisp + +You can type @key{C-x w s} to toggle the side windows to hide or show them at +will. + +Note that you may need to customize @code{mu4e-view-rendered-hook} as well; by +default it contains @code{mu4e-resize-linked-headers-window} but you can set it +to @code{nil} if you want to handle manually (through +@code{display-buffer-alist}. + +@node Message view +@chapter The message view + +This chapter discusses the message view, the view for reading e-mail messages. + +After selecting a message in the @ref{Headers view}, it appears in a +message view window, which shows the message headers, followed by the +message body. Its major mode is @code{mu4e-view-mode}, which derives +from @t{gnus-article-mode}. + +@menu +* Overview: MSGV Overview. What is the Message View +* Keybindings: MSGV Keybindings. Do things with your keyboard +* Rich-text and images: MSGV Rich-text and images. Reading rich-text messages +* Attachments and MIME-parts: MSGV Attachments and MIME-parts. Working with attachments and other MIME parts +* Custom headers: MSGV Custom headers. Your very own headers +* Actions: MSGV Actions. Defining and using actions +* Detaching & reattaching: MSGV Detaching and reattaching. Multiple message views. +@end menu + +@node MSGV Overview +@section Overview + +An example message view: + +@cartouche +@verbatim + From: randy@epiphyte.com + To: julia@eruditorum.org + Subject: Re: some pics + Flags: seen, attach + Date: Thu, 11 Feb 2021 12:59:30 +0200 (4 weeks, 3 days, 21 hours ago) + Maildir: /inbox + Attachments: [2. image/jpeg; DSCN4961.JPG]... [3. image/jpeg; DSCN4962.JPG]... + + Hi Julia, + + Some pics from our trip to Cerin Amroth. Enjoy! + + All the best, + Randy. + + On Sun 21 Dec 2003 09:06:34 PM EET, Julia wrote: + + [....] +@end verbatim +@end cartouche + +Some notes: +@itemize +@item The variable @code{mu4e-view-fields} determines the header fields to be +shown; see @code{mu4e-header-info} for a list of built-in fields. Apart +from the built-in fields, you can also create custom fields using +@code{mu4e-header-info-custom}; see @ref{MSGV Custom headers}. +@item For search-related operations, see @ref{Searching}. +@item You can scroll down the message using @key{SPC}; if you do this at the +end of a message,it automatically takes you to the next one. If you want +to prevent this behavior, set @code{mu4e-view-scroll-to-next} to +@code{nil}. +@end itemize + +@node MSGV Keybindings +@section Keybindings + +You can find most things you can do with this message in the @emph{View} menu, +or by using the keyboard; the default bindings are: + +@verbatim +key description +============================================================== +n,p view the next, previous message +],[ move to the next, previous unread message +},{ move to the next, previous thread +y select the headers view (if visible) + +RET scroll down +M-RET open URL at point / attachment at point + +SPC scroll down, if at end, move to next message +S-SPC scroll up + +searching +--------- +s search +S edit last query +/ narrow the search +b search bookmark +B edit bookmark before search +c search query with completion +j jump to maildir + +O change sort order +P toggle search property + +M-left previous query +M-right next query + +marking messages +---------------- +d mark for moving to the trash folder += mark for removing trash flag ('untrash') +DEL,D mark for complete deletion +m mark for moving to another maildir folder +r mark for refiling ++,- mark for flagging/unflagging + +u unmark message at point +U unmark *all* messages + +% mark based on a regular expression +T,t mark whole thread, subthread + +<insert>,* mark for 'something' (decide later) +# resolve deferred 'something' marks + +x execute actions for the marked messages + +composition +----------- +R,W,F,C reply/reply-to-all/forward/compose +E edit (only allowed for draft messages) + +actions +------- +g go to (visit) numbered URL (using `browse-url') +(or: <mouse-2> or M-RET with point on URL) +C-u g visits multiple URLs +f fetch (download )the numbered URL. +C-u f fetches multiple URLs +k save the numbered URL in the kill-ring. +C-u k saves multiple URLs + +e extract (save) one or more attachments (asks for numbers) +(or: <mouse-2> or S-RET with point on attachment) +a execute some custom action on the message +A execute some custom action on the message's MIME-parts + +misc +---- +z, Z detach (or reattach) a message view to a headers buffer +. show the raw message view. 'q' takes you back. +C-+,C-- increase / decrease the number of headers shown +H get help +C-S-u update mail & reindex +q leave the message view +@end verbatim + +Some keybindings are available through minor modes: +@itemize +@item Context; see @pxref{Contexts} +@item Composition; see @pxref{Composer} and @t{mu4e-compose-minor-mode} +@end itemize + +For the marking commands, please refer to @ref{Marking messages}. + +@node MSGV Rich-text and images +@section Reading rich-text messages +@cindex rich-text + +These days, many e-mail messages contain rich-text (typically, HTML); +either as an alternative to a text-only version, or even as the only +option. + +By default, mu4e tries to display the 'richest' option, which is the +last MIME-part of the alternatives. You can customize this to prefer +the text version, if available, with something like the following in +your configuration (and see the docstring for +@t{mm-discouraged-alternatives} for details): + +@lisp +(with-eval-after-load "mm-decode" + (add-to-list 'mm-discouraged-alternatives "text/html") + (add-to-list 'mm-discouraged-alternatives "text/richtext")) +@end lisp + +When displaying rich-text messages inline, @t{mu4e} (through @t{gnus}) +uses the @t{shr} built-in HTML-renderer. If you're using a dark color +theme, and the messages are hard to read, it can help to change the +luminosity, e.g.: +@lisp +(setq shr-color-visible-luminance-min 80) +@end lisp + +Note that you can switch between the HTML and text versions by +clicking on the relevant part in the messages headers; you can make it +even clearer by indicating them in the message itself, using: + +@lisp +(setq gnus-unbuttonized-mime-types nil) +@end lisp + +@subsection Inline images +When you run Emacs in graphical mode, by default images attached to +messages are shown inline in the message view buffer. + +To disable this, set @code{gnus-inhibit-images} to @t{t}. By default, +external images in HTML are not retrieved from external URLs because +they can be used to track you. + +Apart from that, you can also control whether to load remote images; +since loading remote images is often used for privacy violations, by +default this is not allowed. + +You can specify what URLs to block by setting +@code{gnus-blocked-images} to a regular expression or to a function +that will receive a single parameter which is not meaningful for +@t{mu4e}. + +For example, to enable images in Github notifications, you could use +the following: + +@lisp +(setq gnus-blocked-images + (lambda(&optional _ignore) + (if (mu4e-message-contact-field-matches + (mu4e-message-at-point) :from "notifications@@github.com") + nil "."))) +@end lisp + +@code{mu4e} inherits the default @t{gnus-blocked-images} from Gnus and +ensures that it works with @t{mu4e} too. However, mu4e is not Gnus, so +if you have Gnus-specific settings for @t{gnus-blocked-images}, you +should verify that they have the desired effect in @code{mu4e} as +well. + +@node MSGV Attachments and MIME-parts +@section Attachments and MIME-parts +@cindex attachments +@cindex mime-parts + +E-mail messages can be though as a series of ``MIME-parts'', which are sections +of the message. The most prominent is the 'body', that is the main message your +are reading. Many e-mail messages also contains @emph{attachments}, which +MIME-parts that contain files@footnote{Attachments come in two flavors: +@c{inline} and @c{attachment}. @t{mu4e} does not distinguish between them when +operating on them; everything that specifies a filename is considered an +attachment}. + +To save such attachments as files on your file systems, the @t{mu4e} +message-view offers the command @code{mu4e-view-save-attachments}; default +keybinding is @key{e} (think @emph{extract}). After invoking the command, you +can enter the file names to save, comma-separated, and using the completion +support. Press @key{RET} to save the chosen files to your file-system. + +With a prefix argument, you get to choose the target-directory, otherwise, +@t{mu4e} determines it following the variable @t{mu4e-attachment-dir} (which can +be file-system path or a function; see its docstring for details. + +While completing, @code{mu4e-view-completion-minor-mode} is active, which offers +@code{mu4e-view-complete-all} (bound to @key{C-c C-a} to complete @emph{all} +files@footnote{Except when using 'Helm'; in that case, use the Helm-mechanism +for selecting multiple}. + +@subsection MIME-parts + +Not all MIME-parts are message bodies or attachments, and it can be useful to +operate on those other parts as well. For that, there is the function +@code{mu4e-view-mime-part-action} (default key-binding @key{A}). You can pass +the number of the MIME-pars (as seen in the message view) as a prefix argument, +otherwise you get to get to choose from a completion menu. + +After choosing one or more MIME-parts, you are asked for an action to apply to +them; see the variable @code{mu4e-view-mime-part-actions} for the possibilities; +and you can add your own actions as well, see @ref{MIME-part actions} for some +example. + +@node MSGV Custom headers +@section Custom headers +@cindex custom headers + +Sometimes the normal headers (Date, From, To, Subject, etc.)@: may not be +enough. For these cases, @t{mu4e} offers @emph{custom headers} in both the +headers-view and the message-view. + +See @ref{HV Custom headers} for an example of this; the difference for +the message-view is that you should add your custom header to +@code{mu4e-view-fields} rather than @code{mu4e-headers-fields}. + +@node MSGV Actions +@section Actions + +You can perform custom functions (``actions'') on messages and their +attachments. For a general discussion on how to define your own, see +@ref{Actions}. + +@subsection Message actions +@code{mu4e-view-action} (@key{a}) lets you pick some custom action to perform +on the current message. You can specify these actions using the variable +@code{mu4e-view-actions}; @t{mu4e} defines a number of example actions. + +@subsection MIME-part actions +MIME-part actions allow you to act upon MIME-parts in a message - such +as attachments. For now, these actions are defined and documented in +@code{mu4e-view-mime-part-actions} and see + +@node MSGV Detaching and reattaching +@section Detaching and reattaching messages + +You can have multiple message views, but you must rename the view +buffer and detach it to stop @t{mu4e} from reusing it when you +navigate up or down in the headers buffer. If you have several view +buffers attached to a headers view, then @t{mu4e} may pick one at +random when it has to choose which one to display a message in. + +To detach the message view from its linked headers buffer, type +@key{z}. A message will appear saying it is detached (or warn you if +it is already detached.) + +Detached buffers are static; they cannot change the displayed message, +and no headers buffer will use a detached buffer to display its +messages. You can reattach a buffer to an live headers buffer by +typing @key{Z}. + +You can freely rename a message view buffer -- such as with @key{C-x x +r} -- if you want a custom, non-randomized name. + +Detached messages are often useful for workflows involving lots of +simultaneous messages. + +You can @emph{tear off} the window a message is in and place it in a +new frame by typing @key{C-x w ^ f}. You can also detach a window and +put it in its own tab with @key{C-x w ^ t}. + +@node Composer +@chapter Composer + +Writing e-mail messages takes place in the Composer. @t{mu4e}'s re-uses much of +Gnus' @t{message-mode}. + +Much of the @t{message-mode} functionality is available, as well some +@t{mu4e}-specifics. See @ref{(message) Top} for details; not every setting is +necessarily also supported in @t{mu4e}. + +The major mode for the composer is @code{mu4e-compose-mode}. + +@menu +* Composer overview: Composer overview. What is the composer good for +* Entering the composer:: How to start writing messages +* Keybindings: Composer Keybindings. Doing things with your keyboard +* Address autocompletion:: Quickly entering known addresses +* Compose hooks::Calling functions when composing +* Signing and encrypting:: Support for cryptography +* Queuing mail:: Sending mail when the time is ripe +* Message signatures:: Adding your personal footer to messages +* Other settings::Miscellaneous +@end menu + +@node Composer overview +@section Overview + +@cartouche +@verbatim + From: Rupert the Monkey <rupert@example.com> + To: Wally the Walrus <wally@example.com> + Subject: Re: Eau-qui d'eau qui? + --text follows this line-- + + On Mon 16 Jan 2012 10:18:47 AM EET, Wally the Walrus wrote: + + > Hi Rupert, + > + > Dude - how are things? + > + > Later -- Wally. +@end verbatim +@end cartouche + +@node Entering the composer +@section Entering the composer + +There are a view different ways to @emph{enter} the composer; i.e., from other +@t{mu4e} views or even completely outside. + +If you want the composer to start in a new frame or window, you can configure +the variable @t{mu4e-compose-switch}; see its docstring for details. + +@subsection New message + +You can start composing a completely new message with @t{mu4e-compose-new} (with +@kbd{N} from within @t{mu4e}. + +@subsection Reply + +You can compose a reply to an existing message with @t{mu4e-compose-reply} (with +@kbd{R} from within the headers view or when looking at some specific message. + +When you want to reply to @emph{all} recipients of a message, you can use +@t{mu4e-compose-wide-reply}, bound to @kbd{W}. This is often called +``reply-to-all'', while Gnus uses the term ``wide reply''. + +By default, the reply will cite the message being replied to. If you do not want +that, you can set (or @t{let}-bind) @t{message-cite-function} to +@t{mu4e-message-cite-nothing}. + +See @ref{(message) Reply} and @ref{(message) Wide Reply} for further +information. + +Note: in older versions, @t{mu4e-compose-reply} would @emph{ask} whether you +want to reply-to-all or not; if you are nostalgic for that old behavior, you +could add something like the following to your configuration: +@lisp +(defun compose-reply-wide-or-not-please-ask () + "Ask whether to reply-to-all or not." + (interactive) + (mu4e-compose-reply (yes-or-no-p "Reply to all?"))) + +(define-key mu4e-compose-minor-mode-map (kbd "R") + #'compose-reply-wide-or-not-please-ask) +@end lisp + +@subsection Forward + +You can forward some existing message with @t{mu4e-compose-forward} (with +@kbd{F} from within the headers view or when looking at some specific message. + +For more information, see @ref{(message) Forwarding}. + +To influence the way a message is forwarded, you can use the variables +@code{message-forward-as-mime} and @code{message-forward-show-mml}. + +@subsection Supersede + +Occasionally, it can be useful to ``supersede'' a message you sent; this drops +you into a new message that is just like the old message (and a @t{Supersedes:} +message header). You can then edit this message and send it. + +This is only possible for messages @emph{you} sent, as determined by +@code{mu4e-personal-or-alternative-address-p}. + +This wraps @code{message-supersede}. + +@subsection Resend + +You can re-send some existing message with @t{mu4e-compose-resend} from within +the headers view or when looking at some specific message. + +This re-sends the message without letting you edit it, as per @ref{(message) +Resending}. + + +@node Composer Keybindings +@section Keybindings + +@t{mu4e}'s composer derives from Gnus' message editor and shares most of +its keybindings. Here are some of the more useful ones (you can use the menu +to find more): + +@verbatim +key description +--- ----------- +C-c C-c send message +C-c C-d save to drafts and leave +C-c C-k kill the message buffer (the message remains in the draft folder) +C-c C-a attach a file (pro-tip: drag & drop works as well in graphical context) +C-c C-; switch the context + +(mu4e-specific) +C-S-u update mail & re-index +@end verbatim + +@node Address autocompletion +@section Address autocompletion + +@t{mu4e} supports autocompleting addresses when composing e-mail messages. +@t{mu4e} uses the e-mail addresses from the messages you sent or received as the +source for this. Address auto-completion is enabled by default; if you want to +disable it for some reason, set @t{mu4e-compose-complete-addresses} to @t{nil}. + +This uses the Emacs machinery for showing and cycling through the candidate +addresses; it is active when looking at one of the contact fields in the message +header area. + +It is also possible to use @t{mu4e}'s completion elsewhere in @t{emacs}. To +enable that, a function @t{mu4e-complete-contact} exists, which you can add to +@t{completion-at-point-functions}, see @ref{(elisp) Completion in Buffers}. +@t{mu4e} must be running for any completions to be available. + +@subsection Limiting the number of addresses + +If you have a lot of mail, especially from mailing lists and the like, there +can be a @emph{lot} of e-mail addresses, many of which may not be very useful +when auto-completing. For this reason, @t{mu4e} attempts to limit the number +of e-mail addresses in the completion pool by filtering out the ones that are +not likely to be relevant. The following variables are available for tuning +this: + +@itemize +@item @code{mu4e-compose-complete-only-personal} --- when set to @t{t}, +only consider addresses that were seen in @emph{personal} messages --- +that is, messages in which one of my e-mail addresses was seen in one +of the address fields. This is to exclude mailing list posts. You can +define what is considered `my e-mail address' using the +@t{--my-address} parameter to @t{mu init}. + +@item @code{mu4e-compose-complete-only-after} --- only consider e-mail +addresses last seen after some date. Parameter is a string, parseable by +@code{org-parse-time-string}. This excludes old e-mail addresses. The +default is @t{"2010-01-01"}, i.e., only consider e-mail addresses seen +since the start of 2010. +@item @code{mu4e-compose-complete-max} -- the maximum number of contacts to use. +This adds a hard limit to the 2000 (default) contacts; those are sorted by +recency / frequency etc. so should include the ones you most likely need. +@item @code{mu4e-contact-process-function} --- a function to rewrite or +exclude certain addresses. +@end itemize + +@node Compose hooks +@section Compose hooks + +If you want to change some setting, or execute some custom action before +message composition starts, you can define a @emph{hook function}. @t{mu4e} +offers two hooks: +@itemize +@item @code{mu4e-compose-pre-hook}: this hook is run @emph{before} composition +starts; if you are composing a @emph{reply}, @emph{forward} a message, or +@emph{edit} an existing message, the variable +@code{mu4e-compose-parent-message} points to the message being replied to, +forwarded or edited, and you can use @code{mu4e-message-field} to get the +value of various properties (and see @ref{Message functions}). +@item @code{mu4e-compose-mode-hook}: this hook is run just before composition +starts, when the whole buffer has already been set up. This is a good place +for editing-related settings. @code{mu4e-compose-parent-message} (see above) +is also at your disposal. +@item @code{mu4e-compose-post-hook}: this hook is run when we're done with +message compositions. See the docstring for details. +@end itemize + +@noindent +As mentioned, @code{mu4e-compose-mode-hook} is especially useful for +editing-related settings: + +Let's look at an example: +@lisp +(add-hook 'mu4e-compose-mode-hook + (defun my-do-compose-stuff () + "My settings for message composition." + (set-fill-column 72) + (flyspell-mode))) +@end lisp + +The hook is also useful for adding headers or changing headers, since the +message is fully formed when this hook runs. For example, to add a +@t{Bcc:}-header, you could add something like the following, using +@code{message-add-header} from @code{message-mode}. + +@lisp +(add-hook 'mu4e-compose-mode-hook + (defun my-add-bcc () + "Add a Bcc: header." + (save-excursion (message-add-header "Bcc: me@@example.com\n")))) +@end lisp + +Or to something context-specific: + +@lisp +(add-hook 'mu4e-compose-mode-hook + (lambda() + (let* ((ctx (mu4e-context-current)) + (name (if ctx (mu4e-context-name ctx)))) + (when name + (cond + ((string= name "account1") + (save-excursion (message-add-header "Bcc: account1@@example.com\n"))) + ((string= name "account2") + (save-excursion (message-add-header "Bcc: account2@@example.com\n")))))))) +@end lisp + +@noindent +For a more general discussion about extending @t{mu4e}, see @ref{Extending +mu4e}. + +@node Signing and encrypting +@section Signing and encrypting + +Signing and encrypting of messages is possible using @ref{(emacs-mime) Top, +emacs-mime}, most easily accessed through the @t{Attachments}-menu while +composing a message, or with @kbd{M-x mml-secure-message-encrypt-pgp}, @kbd{M-x +mml-secure-message-sign-pgp}. + +Important note: the messages are encrypted when they are @emph{sent}: this means +that draft messages are @emph{not} encrypted. So if you are using e.g. +@t{offlineimap} or @t{mbsync} to synchronize with some remote IMAP-service, make +sure the drafts folder is @emph{not} in the set of synchronized folders, for +obvious reasons. + +@node Queuing mail +@section Queuing mail + +If you cannot send mail right now, for example because you are +currently offline, you can @emph{queue} the mail, and send it when you +have restored your internet connection. You can control this from the +@ref{Main view}. + +To allow for queuing, you need to tell @t{smtpmail} where you want to store +the queued messages. For example: + +@lisp +(setq smtpmail-queue-mail t ;; start in queuing mode + smtpmail-queue-dir "~/Maildir/queue/cur") +@end lisp + +For convenience, we put the queue directory somewhere in our normal +maildir. If you want to use queued mail, you should create this directory +before starting @t{mu4e}. The @command{mu mkdir} command may be useful here, +so for example: + +@verbatim + $ mu mkdir ~/Maildir/queue + $ touch ~/Maildir/queue/.noindex +@end verbatim + +The file created by the @command{touch} command tells @t{mu} to ignore this +directory for indexing, which makes sense since it contains @t{smtpmail} +meta-data rather than normal messages; see the @t{mu-mkdir} and @t{mu-index} +man-pages for details. + +@emph{Warning}: when you switch on queued-mode, your messages @emph{won't} +reach their destination until you switch it off again; so, be careful not to +do this accidentally! + +@node Message signatures +@section Message signatures + +Message signatures are the standard footer blobs in e-mail messages where you +can put in information you want to include in every message. The text to include +is set with @code{message-signature} (older @t{mu4e} used +@code{mu4e-compose-signature}, but that has been obsoleted). + +@node Other settings +@section Other settings + +@itemize +@item If you want use @t{mu4e} as Emacs' default program for sending mail, +see @ref{Default email client}. +@item Normally, @t{mu4e} @emph{buries} the message buffer after sending; if you want +to kill the buffer instead, add something like the following to your +configuration: +@lisp +(setq message-kill-buffer-on-exit t) +@end lisp +@item If you want to exclude your own e-mail addresses when ``replying to +all'', set @code{message-dont-reply-to-names} to +@code{mu4e-personal-or-alternative-address-p}. In order for this to work +properly you need to pass your address to @command{mu init --my-address=} at +database initialization time, and/or use @t{message-alternative-emails}. +@end itemize + +@node Searching +@chapter Searching + +@t{mu4e} is fully search-based: even if you `jump to a folder', you are +executing a query for messages that happen to have the property of being in a +certain folder (maildir). + +Normally, queries return up to @code{mu4e-headers-results-limit} (default: 500) +results. That is usually more than enough, and makes things significantly +faster. Sometimes, however, you may want to show @emph{all} results; you can +enable this with @kbd{M-x mu4e-headers-toggle-property}, or by customizing the +variable @code{mu4e-headers-full-search}. This applies to all search commands. + +You can also influence the sort order and whether threads are shown or not; +see @ref{Sorting and threading}. + +@menu +* Queries:: Searching for messages. +* Bookmarks:: Remembering queries. +* Maildir searches:: Queries for maildirs. +* Other search functionality:: Some more tricks. +@end menu + +@node Queries +@section Queries + +@t{mu4e} queries are the same as the ones that @t{mu find} +understands@footnote{with the caveat that command-line queries are +subject to the shell's interpretation before @t{mu} sees them}. You can +consult the @code{mu-query} man page for the details. + +Additionally, @t{mu4e} supports @kbd{TAB}-completion for queries. There +there is completion for all search keywords such as @code{and}, +@code{from:}, or @code{date:} and also for certain values, i.e., the +possible values for @code{flag:}, @code{prio:}, @code{mime:}, and +@code{maildir:}. + +Let's look at some examples here. + +@itemize + +@item Get all messages regarding @emph{bananas}: +@verbatim +bananas +@end verbatim + +@item Get all messages regarding @emph{bananas} from @emph{John} with an attachment: +@verbatim +from:john and flag:attach and bananas +@end verbatim + +@item Get all messages with subject @emph{wombat} in June 2017 +@verbatim +subject:wombat and date:20170601..20170630 +@end verbatim + +@item Get all messages with PDF attachments in the @t{/projects} folder +@verbatim +maildir:/projects and mime:application/pdf +@end verbatim + +@item Get all messages about @emph{Rupert} in the @t{/Sent Items} folder. Note that +maildirs with spaces must be quoted. +@verbatim +"maildir:/Sent Items" and rupert +@end verbatim + +@item Get all important messages which are signed: +@verbatim +flag:signed and prio:high +@end verbatim + +@item Get all messages from @emph{Jim} without an attachment: +@verbatim +from:jim and not flag:attach +@end verbatim + +@item Get all messages with Alice in one of the contacts-fields (@t{to}, @t{from}, +@t{cc}, @t{bcc}): +@verbatim +contact:alice +@end verbatim + +@item Get all unread messages where the subject mentions Ångström: (search is +case-insensitive and accent-insensitive, so this matches Ångström, angstrom, +aNGstrøM, ...) +@verbatim +subject:Ångström and flag:unread +@end verbatim + +@item Get all unread messages between Mar-2012 and Aug-2013 about some bird: +@verbatim +date:20120301..20130831 and nightingale and flag:unread +@end verbatim + +@item Get today's messages: +@verbatim +date:today..now +@end verbatim + +@item Get all messages we got in the last two weeks regarding @emph{emacs}: +@verbatim +date:2w.. and emacs +@end verbatim + +@item Get messages from the @emph{Mu} mailing list: +@verbatim +list:mu-discuss.googlegroups.com +@end verbatim + +Note --- in the @ref{Headers view} you may see the `friendly name' for a +list; however, when searching you need the real name. You can see the +real name for a mailing list from the friendly name's tool-tip. + +@item Get messages with a subject soccer, Socrates, society, ...; note that +the `*'-wildcard can only appear as a term's rightmost character: +@verbatim +subject:soc* +@end verbatim + +@item Get all messages @emph{not} sent to a mailing-list: +@verbatim +NOT flag:list +@end verbatim + +@item Get all mails with attachments with filenames starting with @emph{pic}; note +that the `*' wildcard can only appear as the term's rightmost character: +@verbatim +file:pic* +@end verbatim + +@item Get all messages with PDF-attachments: +@verbatim +mime:application/pdf +@end verbatim + +Get all messages with image attachments, and note that the `*' wildcard can +only appear as the term's rightmost character: +@verbatim +mime:image/* +@end verbatim + +Get all messages with files that end in @t{.ppt}; this uses the +regular-expression support, which is powerful but relatively slow: +@verbatim +file:/\.ppt$/ +@end verbatim + +@end itemize + +@node Bookmarks +@section Bookmarks + +If you have queries that you use often, you may want to store them as +@emph{bookmarks}. Bookmark searches are available in the main view +(@pxref{Main view}), header view (@pxref{Headers view}), and message +view (@pxref{Message view}), using (by default) the key @key{b} +(@kbd{M-x mu4e-search-bookmark}), or @key{B} (@kbd{M-x +mu4e-search-bookmark-edit}) which lets you edit the bookmark first. + +@subsection Setting up bookmarks + +@t{mu4e} provides a number of default bookmarks. Their definition may +be instructive: + +@lisp +(defcustom mu4e-bookmarks + '(( :name "Unread messages" + :query "flag:unread AND NOT flag:trashed" + :key ?u) + ( :name "Today's messages" + :query "date:today..now" + :key ?t) + ( :name "Last 7 days" + :query "date:7d..now" + :hide-unread t + :key ?w) + ( :name "Messages with images" + :query "mime:image/*" + :key ?p)) + "List of pre-defined queries that are shown on the main screen. + +Each of the list elements is a plist with at least: +:name - the name of the query +:query - the query expression +:key - the shortcut key. + +Optionally, you add the following: +:hide - if t, bookmark is hidden from the main-view and speedbar. +:hide-unread - do not show the counts of unread/total number + of matches for the query. This can be useful if a bookmark uses + a very slow query. :hide-unread is implied from :hide. +" + :type '(repeat (plist)) + :group 'mu4e) +@end lisp + +You can replace these or add your own items, by putting in your +configuration (@file{~/.emacs}) something like: +@lisp +(add-to-list 'mu4e-bookmarks + '( :name "Big messages" + :query "size:5M..500M" + :key ?b)) + @end lisp + +This prepends your bookmark to the list, and assigns the key @key{b} to it. If +you want to @emph{append} your bookmark, you can use @code{t} as the third +argument to @code{add-to-list}. + +In the various @t{mu4e} views, pressing @key{b} lists all the bookmarks +defined in the echo area, with the shortcut key highlighted. So, to invoke the +bookmark we just defined (to get the list of "Big Messages"), all you need to +type is @kbd{bb}. + +@subsection Lisp expressions or functions as bookmarks + +Instead of using strings, it is also possible to use Lisp expressions as +bookmarks. Either the expression evaluates to a query string or the expression +is a function taking no argument that returns a query string. + +For example, to get all the messages that are at most a week old in your +inbox: + +@lisp +(add-to-list 'mu4e-bookmarks + '( :name "Inbox messages in the last 7 days" + :query (lambda () (concat "maildir:/inbox AND date:" + (format-time-string "%Y%m%d" + (subtract-time (current-time) (days-to-time 7))))) + :key ?w) t) +@end lisp + +Another example where the user is prompted how many days old messages should be +shown: + +@lisp +(defun my/mu4e-bookmark-num-days-old-query (days-old) + (interactive (list (read-number "Show days old messages: " 7))) + (let ((start-date (subtract-time (current-time) (days-to-time days-old)))) + (concat "maildir:/inbox AND date:" + (format-time-string "%Y%m%d" start-date)))) + +(add-to-list 'mu4e-bookmarks + `(:name "Inbox messages in the last 7 days" + :query ,(lambda () (call-interactively 'my/mu4e-bookmark-num-days-old-query)) + :key ?o) t) +@end lisp + +It is defining a function to make the code more readable. + +@subsection Editing bookmarks before searching + +There is also @kbd{M-x mu4e-search-bookmark-edit} (key @key{B}), which +lets you edit the bookmarked query before invoking it. This can be useful if +you have many similar queries, but need to change some parameter. For example, +you could have a bookmark @samp{"date:today..now AND "}@footnote{Not a valid +search query by itself}, which limits any result to today's messages. + +@node Maildir searches +@section Maildir searches + +Maildir searches are quite similar to bookmark searches (see @ref{Bookmarks}), +with the difference being that the target is always a maildir --- maildir +queries provide a `traditional' folder-like interface to a search-based e-mail +client. By default, maildir searches are available in the @ref{Main view}, +@ref{Headers view}, and @ref{Message view}, with the key @key{j} +(@code{mu4e-jump-to-maildir}). If a prefix argument is given, the maildir +query can be refined before execution. + +@subsection Setting up maildir shortcuts + +You can search for maildirs like any other message property +(e.g. with a query like @t{maildir:/myfolder}), but since it is so common, +@t{mu4e} offers a shortcut for this. + +For this to work, you need to set the variable +@code{mu4e-maildir-shortcuts} to the list of maildirs you want to have +quick access to, for example: + +@lisp +(setq mu4e-maildir-shortcuts + '( (:maildir "/inbox" :key ?i) + (:maildir "/archive" :key ?a) + (:maildir "/lists" :key ?l) + (:maildir "/work" :key ?w) + (:maildir "/sent" :key ?s) + (:maildir "/lists/project/project_X" :key ?x :name "Project X"))) +@end lisp + +This sets @key{i} as a shortcut for the @t{/inbox} folder --- effectively a +query @t{maildir:/inbox}. There is a special shortcut @key{o} or @key{/} for +@emph{other} (so don't use those for your own shortcuts!), which allows you to +choose from @emph{all} maildirs that you have. There is support for +autocompletion; note that the list of maildirs is determined when @t{mu4e} +starts; if there are changes in the maildirs while @t{mu4e} is running, you +need to restart @t{mu4e}. Optionally, you can specify a name to be displayed +in the main view. + +Each of the folder names is relative to your top-level maildir directory; so +if you keep your mail in @file{~/Maildir}, @file{/inbox} would refer to +@file{~/Maildir/inbox}. With these shortcuts, you can jump around your +maildirs (folders) very quickly --- for example, getting to the @t{/lists} +folder only requires you to type @kbd{jl}, then change to @t{/work} with +@kbd{jw}. + +While in queries you need to quote folder names (maildirs) with spaces in +them, you should @emph{not} quote them when used in +@code{mu4e-maildir-shortcuts}, since @t{mu4e} does that automatically for you. + +The very same shortcuts are used by @kbd{M-x mu4e-mark-for-move} (default +shortcut @key{m}); so, for example, if you want to move a message to the +@t{/archive} folder, you can do so by typing @kbd{ma}. + +@node Other search functionality +@section Other search functionality + +@subsection Navigating through search queries +You can navigate through previous/next queries using +@code{mu4e-headers-query-prev} and @code{mu4e-headers-query-next}, which are +bound to @key{M-left} and @key{M-right}, similar to what some web browsers do. + +@t{mu4e} tries to be smart and not record duplicate queries. Also, the number +of queries remembered has a fixed limit, so @t{mu4e} won't use too much +memory, even if used for a long time. However, if you want to forget +previous/next queries, you can use @kbd{M-x mu4e-headers-forget-queries}. + +@subsection Narrowing search results + +It can be useful to narrow existing search results, that is, to add some +clauses to the current query to match fewer messages. + +For example, suppose you're looking at some mailing list, perhaps by +jumping to a maildir (@kbd{M-x mu4e-headers-jump-to-maildir}, @key{j}) or +because you followed some bookmark (@kbd{M-x mu4e-search-bookmark}, +@key{b}). Now, you want to narrow things down to only those messages that have +attachments. + +This is when @kbd{M-x mu4e-search-narrow} (@key{/}) comes in handy. It +asks for an additional search pattern, which is appended to the current search +query, in effect getting you the subset of the currently shown headers that +also match this extra search pattern. @key{\} takes you back to the previous +query, so, effectively `widens' the search. Technically, narrowing the results +of query @t{x} with expression @t{y} implies doing a search @t{(x) AND (y)}. + +Note that messages that were not in your original search results because of +@code{mu4e-search-results-limit} may show up in the narrowed query. + +@subsection Including related messages +@anchor{Including related messages} + +It can be useful to not only show the messages that directly match a certain +query, but also include messages that are related to these messages. That is, +messages that belong to the same discussion threads are included in the results, +just like e.g. Gmail does it. You can enable this behavior by setting +@code{mu4e-search-include-related} to @code{t}, and you can toggle between +including/not-including using @key{P} (@code{mu4e-search-toggle-property}). + +Be careful though when e.g. deleting ranges of messages from a certain +folder --- the list may now also include messages from @emph{other} +folders. + +@subsection Skipping duplicates +@anchor{Skipping duplicates} + +Another useful feature is skipping of @emph{duplicate messages}. When you have +copies of messages, there's usually little value in including more than one in +search results. A common reason for having multiple copies of messages is the +combination of Gmail and @t{offlineimap}, since that is the way the labels / +virtual folders in Gmail are represented. You can enable skipping duplicates by +setting @code{mu4e-search-skip-duplicates} to @code{t}, and you can toggle +the value using @key{P} (@code{mu4e-search-toggle-property}). + +Note, messages are considered duplicates when they have the same +@t{Message-Id}. + +@node Marking +@chapter Marking + +In @t{mu4e}, the common way to do things with messages is a two-step process - +first you @emph{mark} them for a certain action, then you @emph{execute} +(@key{x}) those marks. This is similar to the way @t{dired} operates. Marking +can happen in both the @ref{Headers view} and the @ref{Message view}. + +@menu +* Marking messages::Selecting message do something with them +* What to mark for::What can we do with them +* Executing the marks::Do it +* Trashing messages::Exceptions for mailboxes like Gmail +* Leaving the headers buffer::Handling marks automatically when leaving +* Built-in marking functions::Helper functions for dealing with them +* Custom mark functions::Define your own mark function +* Adding a new kind of mark::Adding your own marks +@end menu + +@node Marking messages +@section Marking messages + +There are multiple ways to mark messages: +@itemize +@item @emph{message at point}: you can put a mark on the message-at-point in +either the @ref{Headers view} or @ref{Message view} +@item @emph{region}: you can put a mark on all messages in the current region +(selection) in the @ref{Headers view} +@item @emph{pattern}: you can put a mark on all messages in the @ref{Headers +view} matching a certain pattern with @kbd{M-x mu4e-headers-mark-pattern} +(@key{%}) +@item @emph{thread/subthread}: You can put a mark on all the messages in the +thread/subthread at point with @kbd{M-x mu4e-headers-mark-thread} and @kbd{M-x +mu4e-headers-mark-subthread}, respectively +@end itemize + +@node What to mark for +@section What to mark for + +@t{mu4e} supports a number of marks: + +@cartouche +@verbatim +mark for/as | keybinding | description +-------------+-------------+------------------------------ +'something' | *, <insert> | mark now, decide later +delete | D, <delete> | delete +flag | + | mark as 'flagged' ('starred') +move | m | move to some maildir +read | ! | mark as read +refile | r | mark for refiling +trash | d | move to the trash folder +untrash | = | remove 'trash' flag +unflag | - | remove 'flagged' mark +unmark | u | remove mark at point +unmark all | U | remove all marks +unread | ? | marks as unread +action | a | apply some action +@end verbatim +@end cartouche + +After marking a message, the left-most columns in the headers view indicate +the kind of mark. This is informative, but if you mark many (say, thousands) +messages, this slows things down significantly@footnote{this uses an +Emacs feature called @emph{overlays}, which are slow when used a lot +in a buffer}. For this reason, you can disable this by setting +@code{mu4e-headers-show-target} to @code{nil}. + +@t{something} is a special kind of mark; you can use it to mark messages +for `something', and then decide later what the `something' should +be@footnote{This kind of `deferred marking' is similar to the facility +in @t{dired}, @t{midnight commander} +(@url{https://www.midnight-commander.org/}) and the like, and uses the +same key binding (@key{insert}).} Later, you can set the actual mark +using @kbd{M-x mu4e-mark-resolve-deferred-marks} +(@key{#}). Alternatively, @t{mu4e} will ask you when you try to execute +the marks (@key{x}). + +@node Executing the marks +@section Executing the marks + +After you have marked some messages, you can execute them with @key{x} +(@kbd{M-x mu4e-mark-execute-all}). + +A hook, @code{mu4e-mark-execute-pre-hook}, is available which is run +right before execution of each mark. The hook is called with two +arguments, the mark and the message itself. + +@node Trashing messages +@section Trashing messages + +For regular mailboxes, trashing works like other marks: when executed, +the message is flagged as trashed. Depending on your mailbox provider, +the trash flag is used to automatically move the message to the trash +folder (@code{mu4e-trash-folder}) for instance. + +Some mailboxes behave differently however and they don't interpret the +trash flag. In cases like Gmail, the message must be @emph{moved} to +the trash folder and the trash flag must not be used. + +@node Leaving the headers buffer +@section Leaving the headers buffer + +When you quit or update a headers buffer that has marked messages (for +example, by doing a new search), @t{mu4e} asks you what to do with them, +depending on the value of the variable @code{mu4e-headers-leave-behavior} --- +see its documentation. + +@node Built-in marking functions +@section Built-in marking functions + +Some examples of @t{mu4e}'s built-in marking functions. + +@itemize +@item @emph{Mark the message at point for trashing}: press @key{d} +@item @emph{Mark all messages in the buffer as unread}: press @kbd{C-x h o} +@item @emph{Delete the messages in the current thread}: press @kbd{T D} +@item @emph{Mark messages with a subject matching ``hello'' for flagging}: +press @kbd{% s hello RET}. +@end itemize + +@node Custom mark functions +@section Custom mark functions + +Sometimes, the built-in functions to mark messages may not be sufficient for +your needs. For this, @t{mu4e} offers an easy way to define your own custom +mark functions. You can choose one of the custom marker functions by pressing +@key{&} in the @ref{Headers view} and @ref{Message view}. + +Custom mark functions are to be appended to the list +@code{mu4e-headers-custom-markers}. Each of the elements of this list +('markers') is a list with two or three elements: +@enumerate +@item The name of the marker --- a short string describing this marker. The +first character of this string determines its shortcut, so these should be +unique. If necessary, simply prefix the name with a unique character. +@item a predicate function, taking two arguments @code{msg} and @code{param}. +@code{msg} is the message plist (see @ref{Message functions}) and @code{param} is +a parameter provided by the third of the marker elements (see the next +item). The predicate function should return non-@t{nil} if the message +matches. +@item (optionally) a function that is evaluated once, and the result is passed as a +parameter to the predicate function. This is useful when user-input is needed. +@end enumerate + +Let's look at an example: suppose we want to match all messages that have more +than @emph{n} recipients --- we could do this with the following recipe: + +@lisp +(add-to-list 'mu4e-headers-custom-markers + '("More than n recipients" + (lambda (msg n) + (> (+ (length (mu4e-message-field msg :to)) + (length (mu4e-message-field msg :cc))) n)) + (lambda () + (read-number "Match messages with more recipients than: "))) t) +@end lisp + +After evaluating this expression, you can use it by pressing @key{&} in +the headers buffer to select a custom marker function, and then @key{M} +to choose this particular one (@t{M} because it is the first character +of the description). + +As you can see, it's not very hard to define simple functions to match +messages. There are more examples in the defaults for +@code{mu4e-headers-custom-markers}; see @file{mu4e-headers.el} and see +@ref{Extending mu4e} for general information about writing your own functions. + + +@node Adding a new kind of mark +@section Adding a new kind of mark + +It is possible to configure new marks, by adding elements to the list +@code{mu4e-marks}. Such an element must have the following form: + +@lisp +(SYMBOL + :char STRING + :prompt STRING + :ask-target (lambda () TARGET) + :dyn-target (lambda (TARGET MSG) DYN-TARGET) + :show-target (lambda (DYN-TARGET) STRING) + :action (lambda (DOCID MSG DYN-TARGET) nil)) +@end lisp + +The symbol can be any symbol, except for the symbols @code{unmark} and +@code{something}, which are reserved. The rest is a plist with the following +elements: + +@itemize +@item @code{:char} --- the character to display in the headers view. +@item @code{:prompt} --- the prompt to use when asking for marks +(used for example when marking a whole thread). +@item @code{:ask-target} --- a function run once per bulk-operation, and thus suitable for +querying the user about a target for move-like marks. If @t{nil}, the +@t{TARGET} passed to @code{:dyn-target} is @t{nil}. +@item @code{:dyn-target} --- a function run once per message +(The message is passed as @t{MSG} to the function). This function allows +to compute a per-message target, for refile-like marks. If @t{nil}, the +@t{DYN-TARGET} passed to the @code{:action} is the @t{TARGET} obtained as above. +@item @code{:show-target} --- how to display the target in the headers view. +If @code{:show-target} is @t{nil} the @t{DYN-TARGET} is shown (and +@t{DYN-TARGET} must be a string). +@item @code{:action} --- the action to apply on the message when the mark is executed. +@end itemize + +As an example, suppose we would like to add a mark for tagging messages +(GMail-style). We can use the following code (after loading @t{mu4e}): + +@lisp +(add-to-list 'mu4e-marks + '(tag + :char "g" + :prompt "gtag" + :ask-target (lambda () (read-string "What tag do you want to add? ")) + :action (lambda (docid msg target) + (mu4e-action-retag-message msg (concat "+" target))))) +@end lisp + +Adding elements to @code{mu4e-marks} (as in the example) allows you to use the +mark in bulk operations (for example when tagging a whole thread); if you also +want to add a key-binding for the headers view, you can use something like: + +@lisp +(defun my-mu4e-mark-add-tag() + "Add a tag to the message at point." + (interactive) + (mu4e-headers-mark-and-next 'tag)) + +(define-key mu4e-headers-mode-map (kbd "g") #'my-mu4e-mark-add-tag) +@end lisp + +@node Contexts +@chapter Contexts + +@menu +* What are contexts::Defining the concept +* Context policies::How to determine the current context +* Contexts and special folders::Using context variables to determine them +* Contexts example::How to define contexts +@end menu + +It can be useful to switch between different sets of settings in +@t{mu4e}; a typical example is the case where you have different e-mail +accounts for private and work email, each with their own values for +folders, e-mail addresses, mailservers and so on. + +The @code{mu4e-context} system is a @t{mu4e}-specific mechanism to allow +for that; users can define different @i{contexts} corresponding with +groups of setting and either manually switch between them, or let +@t{mu4e} determine the right context based on some user-provided +function. + +Note that there are a number of existing ways to switch accounts in +@t{mu4e}, for example using the method described in the @ref{Tips and +Tricks} section of this manual. Those still work --- but the new mechanism +has the benefit of being a core part of @code{mu4e}, thus allowing for +deeper integration. + +@node What are contexts +@section What are contexts + +Let's see what's contained in a context. Most of it is optional. + +A @code{mu4e-context} is Lisp object with the following members: +@itemize +@item @t{name}: the name of the context, e.g. @t{work} or @t{private} +@item @t{vars}: +an association-list (alist) of variable settings for this account. +@item @t{enter-func}: +an (optional) function that takes no parameter and is invoked when entering +the context. You can use this for extra setup etc. +@item @t{leave-func}: +an (optional) function that takes no parameter and is invoked when leaving +the context. You can use this for clearing things up. +@item @t{match-func}: +an (optional) function that takes an @t{MSG} message plist as argument, +and returns non-@t{nil} if this context matches the situation. @t{mu4e} +uses the first context that matches, in a couple of situations: +@itemize +@item when starting @t{mu4e} to determine the +starting context; in this case, @t{MSG} is nil. You can use e.g. the +host you're running or the time of day to determine which context +matches. +@item before replying to or forwarding a +message with the given message plist as parameter, or @t{nil} when +composing a brand new message. The function should return @t{t} when +this context is the right one for this message, or @t{nil} otherwise. +@item when determining the target folders for deleting, refiling etc; +see @ref{Contexts and special folders}. +@end itemize +@end itemize + +@t{mu4e} uses a variable @code{mu4e-contexts}, which is a list of such +objects. + +@node Context policies +@section Context policies + +When you have defined contexts and you start @t{mu4e} it decides which +context to use based on the variable @code{mu4e-context-policy}; +similarly, when you compose a new message, the context is determined +using @code{mu4e-compose-context-policy}. + +For both of these, you can choose one of the following policies: +@itemize +@item a symbol @code{always-ask}: unconditionally ask the user what context to pick. +@end itemize + +The other choices @b{only apply if none of the contexts match} (i.e., +none of the contexts' match-functions returns @code{t}). We have the +following options: + +@itemize +@item a symbol @code{ask}: ask the user if @t{mu4e} can't figure +things out the context by itself (through the match-function). This is a +good policy if there are no match functions, or if the match functions +don't cover all cases. +@item a symbol @code{ask-if-none}: if there's already a context, don't change it; +otherwise, ask the user. +@item a symbol @code{pick-first}: pick the first (default) context. This is a +good choice if +you want to specify context for special case, and fall back to the first +one if none match. +@item @code{nil}: don't change the context; this is useful if you don't change +contexts very often, and e.g. manually changes contexts with @kbd{M-x +mu4e-context-switch}. +@end itemize + +You can easily switch contexts manually using the @kbd{;} key from +the main screen. + +@node Contexts and special folders +@section Contexts and special folders + +As we discussed in @ref{Folders} and @ref{Dynamic folders}, @t{mu4e} +recognizes a number of special folders: @code{mu4e-sent-folder}, +@code{mu4e-drafts-folder}, @code{mu4e-trash-folder} and +@code{mu4e-refile-folder}. + +When you have a headers-buffer with messages that belong to different +contexts (say, a few different accounts), it is desirable for each of +them to use the specific folders for their own context --- so, for +instance, if you trash a message, it needs to go to the trash-folder for +the account it belongs to, which is not necessarily the current context. + +To make this easy to do, whenever @t{mu4e} needs to know the value for +such a special folder for a given message, it tries to determine the +appropriate context using @code{mu4e-context-determine} (and policy +@t{nil}; see @ref{Context policies}). If it finds a matching context, it +let-binds the @code{vars} for that account, and then determines the +value for the folder. It does not, however, call the @code{enter-func} +or @code{leave-func}, since we are not really switching contexts. + +In practice, this means that as long as each of the accounts has a good +@t{match-func}, all message operations automatically find the +appropriate folders. + +@node Contexts example +@section Example + +Let's explain how contexts work by looking at an example. We define two +contexts, `Private' and `Work' for a fictional user @emph{Alice +Derleth}. + +Note that in this case, we automatically switch to the first context +when starting; see the discussion in the previous section. + +@lisp + + (setq mu4e-contexts + `( ,(make-mu4e-context + :name "Private" + :enter-func (lambda () (mu4e-message "Entering Private context")) + :leave-func (lambda () (mu4e-message "Leaving Private context")) + ;; we match based on the contact-fields of the message + :match-func (lambda (msg) + (when msg + (mu4e-message-contact-field-matches msg + :to "aliced@@home.example.com"))) + :vars '( ( user-mail-address . "aliced@@home.example.com" ) + ( user-full-name . "Alice Derleth" ) + ( message-user-organization . "Homebase" ) + ( message-signature . + (concat + "Alice Derleth\n" + "Lauttasaari, Finland\n")))) + ,(make-mu4e-context + :name "Work" + :enter-func (lambda () (mu4e-message "Switch to the Work context")) + ;; no leave-func + ;; we match based on the maildir of the message + ;; this matches maildir /Arkham and its sub-directories + :match-func (lambda (msg) + (when msg + (string-match-p "^/Arkham" (mu4e-message-field msg :maildir)))) + :vars '( ( user-mail-address . "aderleth@@miskatonic.example.com" ) + ( user-full-name . "Alice Derleth" ) + ( message-user-organization . "Miskatonic University" ) + ( message-signature . + (concat + "Prof. Alice Derleth\n" + "Miskatonic University, Dept. of Occult Sciences\n")))) + + ,(make-mu4e-context + :name "Cycling" + :enter-func (lambda () (mu4e-message "Switch to the Cycling context")) + ;; no leave-func + ;; we match based on the maildir of the message; assume all + ;; cycling-related messages go into the /cycling maildir + :match-func (lambda (msg) + (when msg + (string= (mu4e-message-field msg :maildir) "/cycling"))) + :vars '( ( user-mail-address . "aderleth@@example.com" ) + ( user-full-name . "AliceD" ) + ( message-signature . nil))))) + + ;; set `mu4e-context-policy` and `mu4e-compose-policy` to tweak when mu4e should + ;; guess or ask the correct context, e.g. + + ;; start with the first (default) context; + ;; default is to ask-if-none (ask when there's no context yet, and none match) + ;; (setq mu4e-context-policy 'pick-first) + + ;; compose with the current context is no context matches; + ;; default is to ask + ;; (setq mu4e-compose-context-policy nil) +@end lisp + +A couple of notes about this example: +@itemize +@item You can manually switch the context use @code{M-x mu4e-context-switch}, +by default bound to @kbd{;} in headers, view and main mode. The current context +appears in the modeline by default; see @ref{Modeline} for details. +@item Normally, @code{M-x mu4e-context-switch} does not call the enter or +leave functions if the 'new' context is the same as the old one. +However, with a prefix-argument (@kbd{C-u}), you can force @t{mu4e} to +invoke those function even in that case. +@item The function @code{mu4e-context-current} returns the current-context; +the current context is also visible in the mode-line when in +headers, view or main mode. +@item You can set any kind of variable; including settings for mail servers etc. +However, settings such as @code{mu4e-mu-home} are not changeable after +they have been set without quitting @t{mu4e} first. +@item @code{leave-func} (if defined) for the context we are leaving, is invoked +before the @code{enter-func} (if defined) of the +context we are entering. +@item @code{enter-func} (if defined) is invoked before setting the variables. +@item @code{match-func} (if defined) is invoked just before @code{mu4e-compose-pre-hook}. +@item See the variables @code{mu4e-context-policy} and +@code{mu4e-compose-context-policy} to tweak what @t{mu4e} should do when +no context matches (or if you always want to be asked). +@item Finally, be careful to get the quotations right --- backticks, single quotes +and commas and note the '.' between variable name and its value. +@end itemize + +@node Dynamic folders +@chapter Dynamic folders + +In @ref{Folders}, we explained how you can set up @t{mu4e}'s special +folders: +@lisp +(setq + mu4e-sent-folder "/sent" ;; sent messages + mu4e-drafts-folder "/drafts" ;; unfinished messages + mu4e-trash-folder "/trash" ;; trashed messages + mu4e-refile-folder "/archive") ;; saved messages +@end lisp + +In some cases, having such static folders may not suffice --- perhaps you want +to change the folders depending on the context. For example, the folder for +refiling could vary, based on the sender of the message. + +To make this possible, instead of setting the standard folders to a string, +you can set them to be a @emph{function} that takes a message as its +parameter, and returns the desired folder name. This chapter shows you how to +do that. For a more general discussion of how to extend @t{mu4e} and writing +your own functions, see @ref{Extending mu4e}. + +If you use @t{mu4e-context}, see @ref{Contexts and special folders} for +what that means for these special folders. + +@menu +* Smart refiling:: Automatically choose the target folder +* Other dynamic folders:: Flexible folders for sent, trash, drafts +@end menu + + +@node Smart refiling +@section Smart refiling + +When refiling messages, perhaps to archive them, it can be useful to have +different target folders for different messages, based on some property of +those message --- smart refiling. + +To accomplish this, we can set the refiling folder (@code{mu4e-refile-folder}) +to a function that returns the actual refiling folder for the particular +message. An example should clarify this: + +@lisp +(setq mu4e-refile-folder + (lambda (msg) + (cond + ;; messages to the mu mailing list go to the /mu folder + ((mu4e-message-contact-field-matches msg :to + "mu-discuss@@googlegroups.com") + "/mu") + ;; messages sent directly to some specific address me go to /private + ((mu4e-message-contact-field-matches msg :to "me@@example.com") + "/private") + ;; messages with football or soccer in the subject go to /football + ((string-match "football\\|soccer" + (mu4e-message-field msg :subject)) + "/football") + ;; messages sent by me go to the sent folder + ((mu4e-message-sent-by-me msg + (mu4e-personal-addresses)) + mu4e-sent-folder) + ;; everything else goes to /archive + ;; important to have a catch-all at the end! + (t "/archive")))) +@end lisp + +@noindent +This can be very powerful; you can select some messages in the headers view, +then press @key{r}, and have them all marked for refiling to their particular +folders. + +Some notes: +@itemize +@item We set @code{mu4e-refile-folder} to an anonymous (@t{lambda}) function. This +function takes one argument, a message plist@footnote{a property list +describing a message}. The plist corresponds to the message at point. See +@ref{Message functions} for a discussion on how to deal with them. +@item In our function, we use a @t{cond} control structure; the function +returns the first of the clauses that matches. It's important to make the last +clause a catch-all, so we always return @emph{some} folder. +@item We use +the convenience function @code{mu4e-message-contact-field-matches}, +which evaluates to @code{t} if any of the names or e-mail addresses in a +contact field (in this case, the @t{To:}-field) matches the regular +expression. With @t{mu4e} version 0.9.16 or newer, the contact field can +in fact be a list instead of a single value, such as @code{'(:to :cc)'}. +@end itemize + +@node Other dynamic folders +@section Other dynamic folders + +Using the same mechanism, you can create dynamic sent-, trash-, and +drafts-folders. The message-parameter you receive for the sent and drafts +folder is the @emph{original} message, that is, the message you reply to, or +forward, or edit. If there is no such message (for example when composing a +brand new message) the message parameter is @t{nil}. + +Let's look at an example. Suppose you want a different trash folder for +work-email. You can achieve this with something like: + +@lisp +(setq mu4e-trash-folder +(lambda (msg) +;; the 'and msg' is to handle the case where msg is nil +(if (and msg +(mu4e-message-contact-field-matches msg :to "me@@work.example.com")) +"/trash-work" +"/trash"))) +@end lisp + +@noindent +Good to remember: +@itemize +@item The @code{msg} parameter you receive in the function refers to the +@emph{original message}, that is, the message being replied to or forwarded. +When re-editing a message, it refers to the message being edited. When you +compose a totally new message, the @code{msg} parameter is @code{nil}. +@item When re-editing messages, the value of @code{mu4e-drafts-folder} is ignored. +@end itemize + + +@node Actions +@chapter Actions + +@t{mu4e} lets you define custom actions for messages in @ref{Headers view} +and for both messages and attachments in @ref{Message view}. Custom +actions allow you to easily extend @t{mu4e} for specific needs --- for example, +marking messages as spam in a spam filter or applying an attachment with a +source code patch. + +You can invoke the actions with key @key{a} for actions on messages, and key +@key{A} for actions on attachments. + +For general information extending @t{mu4e} and writing your own functions, see +@ref{Extending mu4e}. + +@menu +* Defining actions::How to create an action +* Headers view actions::Doing things with message headers +* Message view actions::Doing things with messages +* MIME-part actions::Doing things with MIME-parts such as attachments +* Example actions::Some more examples +@end menu + +@node Defining actions +@section Defining actions + +Defining a new custom action comes down to writing an elisp-function to do the +work. Functions that operate on messages receive a @var{msg} parameter, which +corresponds to the message at point. Something like: +@lisp +(defun my-action-func (msg) + "Describe my message function." + ;; do stuff + ) +@end lisp + +@noindent +Functions that operate on attachments receive a @var{msg} parameter, which +corresponds to the message at point, and an @var{attachment-num}, which is the +number of the attachment as seen in the message view. An attachment function +looks like: +@lisp +(defun my-attachment-action-func (msg attachment-num) + "Describe my attachment function." + ;; do stuff + ) +@end lisp + +@noindent +After you have defined your function, you can add it to the list of +actions@footnote{Instead of defining the functions separately, you can +obviously also add a @code{lambda}-function directly to the list; however, +separate functions are easier to change}, either @code{mu4e-headers-actions}, +@code{mu4e-view-actions} or @code{mu4e-view-mime-part-actions}. The +format@footnote{Note, the format of the actions has changed since version +0.9.8.4, and you must change your configuration to use the new format; +@t{mu4e} warns you when you are using the old format.} of each action is a +cons-cell, @code{(DESCRIPTION . VALUE)}; see below for some examples. If your +shortcut is not also the first character of the description, simply prefix the +description with that character. + +Let's look at some examples. + +@node Headers view actions +@section Headers view actions + +Suppose we want to inspect the number of recipients for a message in the +@ref{Headers view}. We add the following to our configuration: + +@lisp +(defun show-number-of-recipients (msg) + "Display the number of recipients for the message at point." + (message "Number of recipients: %d" + (+ (length (mu4e-message-field msg :to)) + (length (mu4e-message-field msg :cc))))) + +;; define 'N' (the first letter of the description) as the shortcut +;; the 't' argument to add-to-list puts it at the end of the list +(add-to-list 'mu4e-headers-actions + '("Number of recipients" . show-number-of-recipients) t) +@end lisp + +After evaluating this, @kbd{a N} in the headers view shows the number of +recipients for the message at point. + +@node Message view actions +@section Message view actions + +As another example, suppose we would like to search for messages by the +sender of the message at point: + +@lisp +(defun search-for-sender (msg) + "Search for messages sent by the sender of the message at point." + (mu4e-search + (concat "from:" + (mu4e-contact-email (car (mu4e-message-field msg :from)))))) + +;; define 'x' as the shortcut +(add-to-list 'mu4e-view-actions + '("xsearch for sender" . search-for-sender) t) +@end lisp + +@indent +If you wonder why we use @code{car}, remember that the @t{From:}-field +is a list of @code{(:name NAME :email EMAIL)} plists; so this code gets +us the e-mail address of the first in the list. @t{From:}-fields rarely +have more that one address. + +@node MIME-part actions +@section MIME-part actions + +Finally, let's define a MIME-part action. + +The following example action counts the number of lines in an attachment, and +defines @key{n} as its shortcut key (the @key{n} is prefixed to the +description). See the the @code{mu4e-view-mime-part-actions} for the details of +the format. + +@lisp +(add-to-list 'mu4e-view-mime-part-actions + ;; count the number of lines in a MIME-part + '(:name "line-count" :handler "wc -l" :receives pipe)) +@end lisp + +Or another one, to import a calendar invitation into the venerable emacs diary: +@lisp +(add-to-list 'mu4e-view-mime-part-actions + ;; import into calendar; + '(:name "dimport-in-diary" :handler (lambda(file) (icalendar-import-file file diary-file)) + :receives temp)) +@end lisp + +@node Example actions +@section Example actions + +@t{mu4e} includes a number of example actions in the file +@file{mu4e-actions.el} in the source distribution (see @kbd{C-h f +mu4e-action-TAB}). For example, for viewing messages in an external web +browser. + +@node Extending mu4e +@chapter Extending mu4e + +@t{mu4e} is designed to be easily extensible --- that is, write your own +emacs-lisp to make @t{mu4e} behave exactly as you want. Here, we provide some +guidelines for doing so. + +@menu +* Extension points::Where to hook into @t{mu4e} +* Available functions::General helper functions +* Message functions::Working with messages +* Contact functions::Working with contacts +* Utility functions::Miscellaneous helpers +@end menu + +@node Extension points +@section Extension points + +There are a number of places where @t{mu4e} lets you plug in your own +functions: +@itemize +@item Custom functions for message header --- see @ref{HV Custom headers} +@item Using message-specific folders for drafts, trash, sent messages and +refiling, based on a function --- see @ref{Dynamic folders} +@item Using an attachment-specific download-directory --- see the +variable @code{mu4e-attachment-dir}. +@item Apply a function to a message in the headers view - +see @ref{Headers view actions} +@item Apply a function to a message in the message view --- +see @ref{Message view actions} +@item Add a new kind of mark for use in the headers view +- see @ref{Adding a new kind of mark} +@item Apply a function to a MIME-part --- see @ref{MIME-part actions} +@item Custom function to mark certain messages --- +see @ref{Custom mark functions} +@item Using various @emph{mode}-hooks, @code{mu4e-compose-pre-hook} (see +@ref{Compose hooks}), @code{mu4e-index-updated-hook} (see @ref{FAQ}) +@end itemize + +@noindent +You can also write your own functions without using the above. If you +want to do so, key useful functions are @code{mu4e-message-at-point} +(see below), @code{mu4e-headers-for-each} (to iterate over all +headers, see its docstring) and @code{mu4e-view-for-each-part} (to +iterate over all parts/attachments, see its docstring). There is also +@code{mu4e-view-for-each-uri} to iterate of all the URIs in the +current message. + +Another useful function is +@code{mu4e-headers-find-if} which searches for a message matching a +certain pattern; again, see its docstring. + +@node Available functions +@section Available functions + +The whole of @t{mu4e} consists of hundreds of elisp functions. However, +the majority of those are for @emph{internal} use only; you can +recognize them easily, because they all start with @code{mu4e~} or +@code{mu4e--}. These functions make all kinds of assumptions, and they +are subject to change, and should therefore @emph{not} be used. The same +is true for @emph{variables} with the same prefix; don't touch them. Let +me repeat that: +@verbatim +Do not use mu4e~... or mu4e-- functions or variables! +@end verbatim + +@noindent +In addition, you should use functions in the right context; functions +that start with @t{mu4e-view-} are only applicable to the message view, +while functions starting with @t{mu4e-headers-} are only applicable to +the headers view. Functions without such prefixes are applicable +everywhere. + +@node Message functions +@section Message functions + +Many functions in @t{mu4e} deal with message plists (property +lists). They contain information about messages, such as sender and +recipient, subject, date and so on. To deal with these plists, there are +a number of @code{mu4e-message-} functions (in @file{mu4e-message.el}), +such as @code{mu4e-message-field} and @code{mu4e-message-at-point}, and +a shortcut to combine the two, @code{mu4e-message-field-at-point}. + +For example, to get the subject of the message at point, in either the headers +view or the message view, you could write: +@lisp +(mu4e-message-field (mu4e-message-at-point) :subject) +@end lisp +@noindent +Note that: +@itemize +@item The contact fields (To, From, Cc, Bcc) are lists of cons-pairs +@code{(name . email)}; @code{name} may be @code{nil}. So, for example: +@lisp + (mu4e-message-field some-msg :to) + ;; => (("Jack" . "jack@@example.com") (nil . "foo@@example.com")) +@end lisp + +If you are only looking for a match in this list (e.g., ``Is Jack one of the +recipients of the message?''), there is a convenience function +@code{mu4e-message-contact-field-matches} to make this easy. +@item The message body is only available in the message view, not in the +headers view. +@end itemize + +Note that in headers-mode, you only have access to a reduced message +plist, without the information about the message-body or mime-parts; +@t{mu4e} does this for performance reasons. And even in view-mode, you +do not have access to arbitrary message-headers. + +However, it is possible to get the information indirectly, using the +raw-message and some third-party tool like @t{procmail}'s @t{formail}: + +@lisp +(defun my-mu4e-any-message-field-at-point (hdr) + "Quick & dirty way to get an arbitrary header HDR at +point. Requires the 'formail' tool from procmail." + (replace-regexp-in-string "\n$" "" + (shell-command-to-string + (concat "formail -x " hdr " -c < " + (shell-quote-argument (mu4e-message-field-at-point :path)))))) +@end lisp + +@node Contact functions +@section Contact functions + +It can sometimes be useful to discard or rewrite the contact information +that @t{mu4e} provides, for example to fix spelling errors, or omit +unwanted contacts. + +To handle this, @t{mu4e} provides @code{mu4e-contact-process-function}, +which, if defined, is applied to each contact. If the result is @t{nil}, +the contact is discarded, otherwise the (modified or not) contact +information is used. + +Each contact is a full e-mail address as you would see in a +contact-field of an e-mail message, e.g., +@verbatim +"Foo Bar" <foo.bar@example.com> +@end verbatim +or +@verbatim +cuux@example.com +@end verbatim + +An example @code{mu4e-contact-process-function} might look like: + +@lisp +(defun my-contact-processor (contact) + (cond + ;; remove unwanted + ((string-match-p "evilspammer@@example.com" contact) nil) + ((string-match-p "noreply" contact) nil) + ;; + ;; jonh smiht --> John Smith + ((string-match "jonh smiht" contact) + (replace-regexp-in-string "jonh smiht" "John Smith" contact)) + (t contact))) + +(setq mu4e-contact-process-function 'my-contact-processor) +@end lisp + + +@node Utility functions +@section Utility functions + +@file{mu4e-utils} contains a number of utility functions; we list a few +here. See their docstrings for details: +@itemize +@item @code{mu4e-read-option}: read one option from a list. For example: +@lisp +(mu4e-read-option "Choose an animal: " +'(("Monkey" . monkey) ("Gnu" . gnu) ("xMoose" . moose))) +@end lisp +The user is presented with: +@example +Choose an animal: [M]onkey, [G]nu, [x]Moose +@end example +@item @code{mu4e-ask-maildir}: ask for a maildir; try one of the +shortcuts (@code{mu4e-maildir-shortcuts}), or the full set of available +maildirs. +@item @code{mu4e-running-p}: return @code{t} if the @t{mu4e} process is +running, @code{nil} otherwise. +@item @code{(mu4e-user-mail-address-p addr)}: return @code{t} if @var{addr} is +one of the user's e-mail addresses (as per @code{(mu4e-personal-addresses)}). +@item @code{mu4e-log} logs to the @t{mu4e} debugging log if it is enabled; +see @code{mu4e-toggle-logging}. +@item @code{mu4e-message}, @code{mu4e-warning}, @code{mu4e-error} are the +@t{mu4e} equivalents of the normal elisp @code{message}, +@code{user-error} and @code{error} functions. +@end itemize + +@node Integration +@chapter Integrating @t{mu4e} with Emacs facilities + +In this chapter, we discuss how you can integrate @t{mu4e} with Emacs in various +ways. Here we focus on Emacs built-ins; for dealing with external tools, +@xref{Other tools}. + +@menu +* Default email client::Making mu4e the default emacs e-mail program +* Modeline::Showing mu4e's status in the modeline +* Desktop notifications::Get desktop notifications for new mail +* Emacs bookmarks::Using Emacs' bookmark system +* Eldoc::Information about the current header in the echo area +* Org-mode links::Adding mu4e to your organized life +* iCalendar::Enabling iCalendar invite processing +* Speedbar::A special frame with your folders +* Dired:: Attaching files using @t{dired} +@end menu + + +@node Default email client +@section Default email client + +Emacs allows you to select an e-mail program as the default program it uses when +you press @key{C-x m} (@code{compose-mail}), call @code{report-emacs-bug} and so +on; see @ref{(emacs) Mail Methods}. + +If you want to use @t{mu4e} for this, you can do so by adding the following +to your configuration: + +@lisp +(setq mail-user-agent 'mu4e-user-agent) +@end lisp + +Similarly, to specify @t{mu4e} as your preferred method for reading +mail, customize the variable @code{read-mail-command}. + +@lisp +(set-variable 'read-mail-command 'mu4e) +@end lisp + +@node Modeline +@section Modeline +@cindex modeline + +One of the most visible ways in which @t{mu4e} integrates with Emacs is through +the @emph{modeline} @xref{Mode Line,,,emacs}. The @t{mu4e} support for that is +handled through a minor-mode @code{mu4e-modeline-mode}, which is enabled by +default when @t{mu4e} is running. + +To completely turn off the modeline support, set @code{mu4e-modeline-support} to +@t{nil} before starting @t{mu4e}. + +@t{mu4e} shares information on the modeline in two ways: +@itemize +@item buffer-specific +@itemize +@item current context (as per @ref{Contexts}) +@item current query parameters (headers-mode only) +@end itemize +@item global: information about the results for the ``favorite query'' +@end itemize + +The global indicators can be disabled by setting @code{mu4e-modeline-show-global} +to @t{nil}. + +All of the bookmark items provide more details in their @code{help-echo}, +i.e., their tooltip. + +@subsection Query parameters bookmark item +The query parameters in the modeline start with the various query flags (such as +some representation of @code{mu4e-search-threads}, @code{mu4e-search-full}; the +@t{help-echo} (tool-tip) has the details. + +The query parameters are followed by the query-string use for the headers-view. +By default, if the query string matches some bookmark, the name of that bookmark +is shown instead of the query it specifies. This can be changed by setting +@code{mu4e-modeline-prefer-bookmark-name} to @t{nil}. + +@cindex favorite bookmark +@subsection Favorite bookmark modeline item +The global modeline contains the results of some specific ``favorite'' bookmark +query from @code{mu4e-bookmarks}. By default, the @emph{first} one in chosen, +but you may want to change that by using the @code{:favorite} property for a +particular query, e.g., as part of your @var{mu4e-bookmarks}: +@example + ;; Monitor the inbox folder in the modeline + (:query "maildir:/inbox" :name "Inbox" :key ?i :favorite t) +@end example + +The results of this query (the last time it was updated) is shown as some +character or emoji (depending on @var{mu4e-use-fancy-chars}) and 2 or 3 numbers, +just like what we saw in @xref{Bookmarks and Maildirs}, e.g., +@example + N:10(+5)/15 +@end example + +@cindex baseline query results +this means there are @emph{10 unread messages}, with @emph{5 new messages since +the baseline}, and @emph{15 messages in total} matching the query. + +You can customize the icon; see @var{mu4e-modeline-all-clear}, +@var{mu4e-modeline-all-read}, @var{mu4e-modeline-unread-items} and +@var{mu4e-modeline-new-items}. + +Due to the way queries work, the modeline is @emph{not} immediately updated when +you read messages; but going back to the main view (with @kbd{M-x mu4e} resets +the counts to latest known ones. When in the main-view, you can use +@code{revert-buffer} (@kbd{g}) to reset the counters explicitly. + +@node Desktop notifications +@section Desktop notifications +@cindex desktop notifications + +Depending on your desktop environment, it is possible to get notification when +there is new mail. + +The default implementation (which you can override) depends on the same system +used for the @xref{Bookmarks and Maildirs}, in the main view and the +@xref{Modeline}, and thus gives updates when there new messages compared to some +``baseline'', as discussed earlier. + +For now, notifications are implemented for desktop environments that support +DBus-based notifications, as per Emacs' notification sub-system @xref{(elisp) +Desktop Notifications}. + +You can enable mu4e's desktop notifications (provided that you are on a +supported system) by setting @code{mu4e-notification-support} to @t{t}. If you +want tweak the details, have a look at @code{mu4e-notification-filter} and +@code{mu4e-notification-function}. + +@node Emacs bookmarks +@section Emacs bookmarks +@cindex Emacs bookmarks + +Note, Emacs bookmarks are not to be confused with mu4e's bookmarks; the former +are a generic linking system across Emacs, while the latter are stored queries +within @t{mu4e}. + +@t{mu4e} supports linking to the message-at-point through the normal Emacs +built-in bookmark system. The links are based on the message's message-id, and +thus the bookmarks stay valid even if you move the message around. + +@node Eldoc +@section Eldoc +@cindex eldoc + +It is possible to get information about the current header in the echo-area. +You can enable this by setting @t{mu4e-eldoc-support} to non-@t{nil}. + +@node Org-mode links +@section Org-mode links + +It can be useful to include links to e-mail messages or search queries +in your org-mode files. @t{mu4e} supports this by default, unless you +set @t{mu4e-support-org} to @code{nil}. + +You can use the normal @t{org-mode} mechanisms to store links: +@kbd{M-x org-store-link} stores a link to a particular message when +you are in @ref{Message view}. When you are in @ref{Headers view}, +@kbd{M-x org-store-link} links to the @emph{query} if +@code{mu4e-org-link-query-in-headers-mode} is non-@code{nil}, and to +the particular message otherwise (which is the default). You can +customize the link description using @code{mu4e-org-link-desc-func}. + +You can insert this link later with @kbd{M-x org-insert-link}. From +@t{org-mode}, you can go to the query or message the link points to +with either @kbd{M-x org-agenda-open-link} in agenda buffers, or +@kbd{M-x org-open-at-point} elsewhere --- both typically bound to +@kbd{C-c C-o}. + +You can also directly @emph{capture} such links --- for example, to +add e-mail messages to your todo-list. For that, @t{mu4e-org} has a +function @code{mu4e-org-store-and-capture}. This captures the +message-at-point (or header --- see the discussion on +@code{mu4e-org-link-query-in-headers-mode} above), then calls +@t{org-mode}'s capture functionality. + +You can add some specific capture-template for this. In your capture +templates, the following mu4e-specific values are available: + +@cartouche +@verbatim +item | description +-----------------------------------------------------+------------------------ +%:date, %:date-timestamp, %:date-timestamp-inactive | date, org timestamps +%:from, %:fromname, %:fromaddress | sender, name/address +%:to, %:toname, %:toaddress | recipient, name/address +%:maildir | maildir for the message +%:message-id | message-id +%:path | file system path +%:subject | message subject +@end verbatim +@end cartouche + +For example, to add a message to your todo-list, and set a deadline +for processing it within two days, you could add this to +@code{org-capture-templates}: + +@lisp + ("P" "process-soon" entry (file+headline "todo.org" "Todo") + "* TODO %:fromname: %a %?\nDEADLINE: %(org-insert-time-stamp (org-read-date nil t \"+2d\"))") +@end lisp + +If you use the functionality a lot, you may want to define +key-bindings for that in headers and view mode: + +@lisp + (define-key mu4e-headers-mode-map (kbd "C-c c") 'mu4e-org-store-and-capture) + (define-key mu4e-view-mode-map (kbd "C-c c") 'mu4e-org-store-and-capture) +@end lisp + +@node iCalendar +@section iCalendar + +When Gnus' article-mode is chosen (@ref{Message view}), it is possible +to view and reply to iCalendar events. To enable this feature, add + +@lisp +(require 'mu4e-icalendar) +(mu4e-icalendar-setup) +@end lisp + +to your configuration. If you want that the original invitation message +be automatically trashed after sending the message created by clicking +on the buttons “Accept”, “Tentative”, or “Decline”, also add: + +@lisp +(setq mu4e-icalendar-trash-after-reply t) +@end lisp + +When you reply to an iCal event, a line may be automatically added to +the diary file of your choice. You can specify that file with + +@lisp +(setq mu4e-icalendar-diary-file "/path/to/your/diary") +@end lisp + +Note that, if the specified file is not your main diary file, add +@t{#include "/path/to/your/diary"} to you main diary file to display +the events. + +To enable optional iCalendar→Org sync functionality, add the following: + +@lisp +(setq gnus-icalendar-org-capture-file "~/org/notes.org") +(setq gnus-icalendar-org-capture-headline '("Calendar")) +(gnus-icalendar-org-setup) +@end lisp + +Both the capture file and the headline(s) inside it must already exist. + +By default, @code{gnus-icalendar-org-setup} adds a temporary capture +template to the variable @code{org-capture-templates}, with the +description ``used by gnus-icalendar-org'', and the shortcut key ``#''. +If you want to use your own template, create it using the same key and +description. This will prevent the temporary one from being installed +next time you @code{gnus-icalendar-org-setup} is called. + +The full default capture template is: + +@lisp +("#" "used by gnus-icalendar-org" entry + (file+olp ,gnus-icalendar-org-capture-file + ,gnus-icalendar-org-capture-headline) + "%i" :immediate-finish t) +@end lisp + +where the values of the variables @code{gnus-icalendar-org-capture-file} +and @code{gnus-icalendar-org-capture-headline} are inserted via macro +expansion. + +If, for example, you wanted to store ical events in a date tree, +prompting for the date, you could use the following: + +@lisp +("#" "used by gnus-icalendar-org" entry + (file+olp+datetree path-to-capture-file) + "%i" :immediate-finish t :time-prompt t) +@end lisp + +Note that the default behaviour for @code{datetree} targets in this +situation is to store the event at the date that you capture it, not at +the date that it is scheduled. That's why I've suggested using the +@code{:timeprompt t} argument. This gives you an opportunity to set the +time to the correct value yourself. + +You can extract the event time directly, and have the @code{org-capture} +functions use that to set the @code{datetree} location: + +@lisp +(defun my-catch-event-time (orig-fun &rest args) + "Set org-overriding-default-time to the start time of the capture event" + (let ((org-overriding-default-time (date-to-time + (gnus-icalendar-event:start (car args))))) + (apply orig-fun args))) + +(advice-add 'gnus-icalendar:org-event-save :around #'my-catch-event-time) +@end lisp + +If you do this, you'll want to omit the @code{:timeprompt t} setting +from your capture template. + +@node Speedbar +@section Speedbar +@cindex speedbar + +@code{speedbar} is an Emacs-extension that shows navigational +information for an Emacs buffer in a separate frame. Using +@code{mu4e-speedbar}, @t{mu4e} lists your bookmarks and maildir +folders and allows for one-click access to them. + +To enable this, add @t{(require 'mu4e-speedbar)} to your configuration; +then, all you need to do to activate it is @kbd{M-x speedbar}. Then, +when then switching to the @ref{Main view}, the speedbar-frame is +updated with your bookmarks and maildirs. + +For speed reasons, the list of maildirs is determined when @t{mu4e} +starts; if the list of maildirs changes while @t{mu4e} is running, you +need to restart @t{mu4e} to have those changes reflected in the speedbar +and in other places that use this list, such as auto-completion when +jumping to a maildir. + +@node Dired +@section Dired +@cindex dired + +It is possible to attach files to @t{mu4e} messages using @t{dired} +(@ref{Dired,,emacs}), using the following steps (based on a post on +the @t{mu-discuss} mailing list by @emph{Stephen Eglen}). + +@lisp +(add-hook 'dired-mode-hook 'turn-on-gnus-dired-mode) +@end lisp + +Then, mark the file(s) in @t{dired} you would like to attach and press +@t{C-c RET C-a}, and you'll be asked whether to attach them to an +existing message, or create a new one. + +@node Other tools +@appendix Other tools + +In this chapter, we discuss some ways in which @t{mu4e} can cooperate +with other tools. + +@menu +* Org-contacts::Hooking up with org-contacts +* BBDB::Hooking up with the Insidious Big Brother Database +* Sauron::Getting new mail notifications with Sauron +* Hydra:: Custom shortcut menus +@end menu + +@node Org-contacts +@section Org-contacts + +Note, @t{mu4e} supports built-in address autocompletion; @ref{Address +autocompletion}, and that is the recommended way to do this. However, it is also +possible to manage your addresses with @t{org-mode}, using +@uref{https://julien.danjou.info/projects/emacs-packages#org-contacts,org-contacts}. + +@t{mu4e-actions} defines a useful action (@ref{Actions}) for adding a +contact based on the @t{From:}-address in the message at point. To +enable this, add to your configuration something like: + +@lisp + (setq mu4e-org-contacts-file <full-path-to-your-org-contacts-file>) + (add-to-list 'mu4e-headers-actions + '("org-contact-add" . mu4e-action-add-org-contact) t) + (add-to-list 'mu4e-view-actions + '("org-contact-add" . mu4e-action-add-org-contact) t) +@end lisp + +@noindent +After this, you should be able to add contacts using @key{a o} in the +headers view and the message view, using the @t{org-capture} mechanism. +Note, the shortcut character @key{o} is due to the first character of +@t{org-contact-add}. + +@node BBDB +@section BBDB + +Note, @t{mu4e} supports built-in address autocompletion; @ref{Address +autocompletion}, and that is the recommended way to do this. However, it is also +possible to manage your addresses with +@uref{https://savannah.nongnu.org/projects/bbdb/,BBDB}. + +To enable BBDB, add to your @file{~/.emacs} (or its moral equivalent, +such as @file{~/.emacs.d/init.el}) the following @emph{after} the +@code{(require 'mu4e)} line: + +@lisp + ;; Load BBDB (Method 1) + (require 'bbdb-loaddefs) + ;; OR (Method 2) + ;; (require 'bbdb-loaddefs "/path/to/bbdb/lisp/bbdb-loaddefs.el") + ;; OR (Method 3) + ;; (autoload 'bbdb-insinuate-mu4e "bbdb-mu4e") + ;; (bbdb-initialize 'message 'mu4e) + + (setq bbdb-mail-user-agent 'mu4e-user-agent) + (setq mu4e-view-rendered-hook 'bbdb-mua-auto-update) + (setq mu4e-compose-complete-addresses nil) + (setq bbdb-mua-pop-up t) + (setq bbdb-mua-pop-up-window-size 5) + (setq mu4e-view-show-addresses t) +@end lisp + +For recent emacs (29 and later), address-completion may need some extra setup: +@lisp +(add-hook 'message-mode-hook + (lambda () + (add-to-list 'completion-at-point-functions + #'eudc-capf-complete))) +@end lisp +or, if that does not work: +@lisp +(add-hook 'message-mode-hook + (lambda () + (add-to-list 'completion-at-point-functions + #'message-expand-name))) +@end lisp + +@noindent +After this, you should be able to: +@itemize +@item In mu4e-view mode, add the sender of the email to BBDB with @key{C-u :} +@item Tab-complete addresses from BBDB when composing emails +@item View the BBDB contact while viewing a message +@end itemize + + + +@node Sauron +@section Sauron + +The Emacs package @uref{https://github.com/djcb/sauron,sauron} (by the same +author) can be used to get notifications about new mails. If you run something +like the below script from your @t{crontab} (or have some other way of having it +execute every @emph{n} minutes), you receive notifications in the +@t{sauron}-buffer when new messages arrive. + +@verbatim +#!/bin/sh + +# the mu binary +MU=mu + +# put the path to your Inbox folder here +CHECKDIR="/home/$LOGNAME/Maildir/Inbox" + +sauron_msg () { +DBUS_COOKIE="/home/$LOGNAME/.sauron-dbus" +if test "x$DBUS_SESSION_BUS_ADDRESS" = "x"; then + if test -e $DBUS_COOKIE; then + export DBUS_SESSION_BUS_ADDRESS="`cat $DBUS_COOKIE`" + fi +fi +if test -n "x$DBUS_SESSION_BUS_ADDRESS"; then + dbus-send --session \ + --dest="org.gnu.Emacs" \ + --type=method_call \ + "/org/gnu/Emacs/Sauron" \ + "org.gnu.Emacs.Sauron.AddMsgEvent" \ + string:shell uint32:3 string:"$1" +fi +} + +# +# -mmin -5: consider only messages that were created / changed in the +# the last 5 minutes +# +for f in `find $CHECKDIR -mmin -5 -a -type f -not -iname '.uidvalidity'`; do + subject=`$MU view $f | grep '^Subject:' | sed 's/^Subject://'` + sauron_msg "mail: $subject" +done +@end verbatim + +@noindent +You might want to put: +@lisp +(setq sauron-dbus-cookie t) +@end lisp +@noindent +in your setup, to allow the script to find the D-Bus session bus, even when +running outside its session. + + +@node Hydra +@section Hydra + +People sometimes ask about having multi-character shortcuts for bookmarks; an +easy way to achieve this, is by using an emacs package +@uref{https://github.com/abo-abo/hydra,Hydra}. + +With Hydra installed, we can add multi-character shortcuts, for instance: +@lisp +(defhydra my-mu4e-bookmarks-work (:color blue) + "work bookmarks" + ("b" (mu4e-search "banana AND maildir:/work") "banana") + ("u" (mu4e-search "flag:unread AND maildir:/work") "unread")) + +(defhydra my-mu4e-bookmarks-personal (:color blue) + "personal bookmarks" + ("c" (mu4e-search "capybara AND maildir:/personal") "capybara") + ("u" (mu4e-search "flag:unread AND maildir:/personal") "unread")) + +(defhydra my-mu4e-bookmarks (:color blue) + "mu4e bookmarks" + ("p" (my-mu4e-bookmarks-personal/body) "Personal") + ("w" (my-mu4e-bookmarks-work/body) "Work")) + +Now, you can bind a convenient key to my-mu4e-bookmarks/body. +@end lisp + +@node Example configurations +@appendix Example configurations + +In this chapter, we show some example configurations. While it is very useful +to see some working settings, we'd like to warn against blindly copying such +things. + +@menu +* Minimal configuration::Simplest configuration to get you going +* Longer configuration::A more extensive setup +* Gmail configuration::GMail-specific setup +* Other settings:CONF Other settings. Some other useful configuration + +@end menu + +@node Minimal configuration +@section Minimal configuration + +An (almost) minimal configuration for @t{mu4e} might look like this --- as you +see, most of it is commented-out. + +@lisp +;; example configuration for mu4e + +;; make sure mu4e is in your load-path +(require 'mu4e) + +;; use mu4e for e-mail in emacs +(setq mail-user-agent 'mu4e-user-agent) + +;; these must start with a "/", and must exist +;; (i.e.. /home/user/Maildir/sent must exist) +;; you use e.g. 'mu mkdir' to make the Maildirs if they don't +;; already exist + +;; below are the defaults; if they do not exist yet, mu4e offers to +;; create them. they can also functions; see their docstrings. +;; (setq mu4e-sent-folder "/sent") +;; (setq mu4e-drafts-folder "/drafts") +;; (setq mu4e-trash-folder "/trash") + +;; smtp mail setting; these are the same that `gnus' uses. +(setq + message-send-mail-function 'smtpmail-send-it + smtpmail-default-smtp-server "smtp.example.com" + smtpmail-smtp-server "smtp.example.com" + smtpmail-local-domain "example.com") +@end lisp + + +@node Longer configuration +@section Longer configuration + +A somewhat longer configuration, showing some more things that you can +customize. + +@lisp +;; example configuration for mu4e +(require 'mu4e) + +;; use mu4e for e-mail in emacs +(setq mail-user-agent 'mu4e-user-agent) + +;; the next are relative to the root maildir +;; (see `mu info`). +;; instead of strings, they can be functions too, see +;; their docstring or the chapter 'Dynamic folders' +(setq mu4e-sent-folder "/sent" + mu4e-drafts-folder "/drafts" + mu4e-trash-folder "/trash") + +;; the maildirs you use frequently; access them with 'j' ('jump') +(setq mu4e-maildir-shortcuts + '((:maildir "/archive" :key ?a) + (:maildir "/inbox" :key ?i) + (:maildir "/work" :key ?w) + (:maildir "/sent" :key ?s))) + +;; the headers to show in the headers list -- a pair of a field +;; and its width, with `nil' meaning 'unlimited' +;; (better only use that for the last field. +;; These are the defaults: +(setq mu4e-headers-fields + '( (:date . 25) ;; alternatively, use :human-date + (:flags . 6) + (:from . 22) + (:subject . nil))) ;; alternatively, use :thread-subject + +(add-to-list 'mu4e-bookmarks + ;; ':favorite t' i.e, use this one for the modeline + '(:query "maildir:/inbox" :name "Inbox" :key ?i :favorite t)) + +;; program to get mail; alternatives are 'fetchmail', 'getmail' +;; isync or your own shellscript. called when 'U' is pressed in +;; main view. + +;; If you get your mail without an explicit command, +;; use "true" for the command (this is the default) +(setq mu4e-get-mail-command "offlineimap") + +;; general emacs mail settings; used when composing e-mail +;; the non-mu4e-* stuff is inherited from emacs/message-mode +(setq mu4e-compose-reply-to-address "foo@@bar.example.com" + user-mail-address "foo@@bar.example.com" + user-full-name "Foo X. Bar") +(setq message-signature "Foo X. Bar\nhttp://www.example.com\n") + +;; smtp mail setting +(setq + message-send-mail-function 'smtpmail-send-it + smtpmail-default-smtp-server "smtp.example.com" + smtpmail-smtp-server "smtp.example.com" + smtpmail-local-domain "example.com" + + ;; if you need offline mode, set these -- and create the queue dir + ;; with 'mu mkdir', i.e.. mu mkdir /home/user/Maildir/queue + smtpmail-queue-mail nil + smtpmail-queue-dir "/home/user/Maildir/queue/cur") + +;; don't keep message buffers around +(setq message-kill-buffer-on-exit t) +@end lisp + + +@node Gmail configuration +@section Gmail configuration + +@emph{Gmail} is a popular e-mail provider; let's see how we can make it +work with @t{mu4e}. Since we are using @abbr{IMAP}, you must enable that +in the Gmail web interface (in the settings, under the ``Forwarding and +POP/IMAP''-tab). + +Gmail users may also be interested in @ref{Including related messages}, +and in @ref{Skipping duplicates}. + +@subsection Setting up offlineimap + +First of all, we need a program to get the e-mail from Gmail to our +local machine; for this we use @t{offlineimap}; on Debian (and +derivatives like Ubuntu), this is as easy as: + +@verbatim +$ sudo apt-get install offlineimap +@end verbatim + +while on Fedora (and similar) you need: +@verbatim +$ sudo yum install offlineimap +@end verbatim + +Then, we can configure @t{offlineimap} by editing @file{~/.offlineimaprc}: + +@verbatim +[general] +accounts = Gmail +maxsyncaccounts = 3 + +[Account Gmail] +localrepository = Local +remoterepository = Remote + +[Repository Local] +type = Maildir +localfolders = ~/Maildir + +[Repository Remote] +type = IMAP +remotehost = imap.gmail.com +remoteuser = USERNAME@gmail.com +remotepass = PASSWORD +ssl = yes +maxconnections = 1 +@end verbatim + +Obviously, you need to replace @t{USERNAME} and @t{PASSWORD} with your actual +Gmail username and password. After this, you should be able to download your +mail: + +@verbatim +$ offlineimap + OfflineIMAP 6.3.4 +Copyright 2002-2011 John Goerzen & contributors. +Licensed under the GNU GPL v2+ (v2 or any later version). + +Account sync Gmail: + ***** Processing account Gmail + Copying folder structure from IMAP to Maildir + Establishing connection to imap.gmail.com:993. +Folder sync [Gmail]: + Syncing INBOX: IMAP -> Maildir + Syncing [Gmail]/All Mail: IMAP -> Maildir + Syncing [Gmail]/Drafts: IMAP -> Maildir + Syncing [Gmail]/Sent Mail: IMAP -> Maildir + Syncing [Gmail]/Spam: IMAP -> Maildir + Syncing [Gmail]/Starred: IMAP -> Maildir + Syncing [Gmail]/Trash: IMAP -> Maildir +Account sync Gmail: + ***** Finished processing account Gmail +@end verbatim + +We can now run @command{mu} to make sure things work: + +@verbatim +$ mu index +mu: indexing messages under /home/foo/Maildir [/home/foo/.cache/mu/xapian] +| processing mail; checked: 520; updated/new: 520, cleaned-up: 0 +mu: elapsed: 3 second(s), ~ 173 msg/s +mu: cleaning up messages [/home/foo/.cache/mu/xapian] +/ processing mail; checked: 520; updated/new: 0, cleaned-up: 0 +mu: elapsed: 0 second(s) +@end verbatim + +We can run both the @t{offlineimap} and the @t{mu index} from within +@t{mu4e}, but running it from the command line makes it a bit easier to +troubleshoot as we are setting things up. + +Note: when using encryption, you probably do @emph{not} want to +synchronize your Drafts-folder, since it contains the unencrypted +messages. You can use OfflineIMAP's @t{folderfilter} for that. + +@subsection Settings + +Next step: let's make a @t{mu4e} configuration for this: + +@lisp +(require 'mu4e) + +;; use mu4e for e-mail in emacs +(setq mail-user-agent 'mu4e-user-agent) + +(setq mu4e-drafts-folder "/[Gmail].Drafts") +(setq mu4e-sent-folder "/[Gmail].Sent Mail") +(setq mu4e-trash-folder "/[Gmail].Trash") + +;; don't save message to Sent Messages, Gmail/IMAP takes care of this +(setq mu4e-sent-messages-behavior 'delete) + +;; (See the documentation for `mu4e-sent-messages-behavior' if you have +;; additional non-Gmail addresses and want assign them different +;; behavior.) + +;; setup some handy shortcuts +;; you can quickly switch to your Inbox -- press ``ji'' +;; then, when you want archive some messages, move them to +;; the 'All Mail' folder by pressing ``ma''. + +(setq mu4e-maildir-shortcuts + '( (:maildir "/INBOX" :key ?i) + (:maildir "/[Gmail].Sent Mail" :key ?s) + (:maildir "/[Gmail].Trash" :key ?t) + (:maildir "/[Gmail].All Mail" :key ?a))) + +(add-to-list 'mu4e-bookmarks + ;; ':favorite t' i.e, use this one for the modeline + '(:query "maildir:/inbox" :name "Inbox" :key ?i :favorite t)) + +;; allow for updating mail using 'U' in the main view: +(setq mu4e-get-mail-command "offlineimap") + +;; something about ourselves +(setq + user-mail-address "USERNAME@@gmail.com" + user-full-name "Foo X. Bar" + message-signature + (concat + "Foo X. Bar\n" + "http://www.example.com\n")) + +;; sending mail -- replace USERNAME with your gmail username +;; also, make sure the gnutls command line utils are installed +;; package 'gnutls-bin' in Debian/Ubuntu + +(require 'smtpmail) +(setq message-send-mail-function 'smtpmail-send-it + starttls-use-gnutls t + smtpmail-starttls-credentials '(("smtp.gmail.com" 587 nil nil)) + smtpmail-auth-credentials + '(("smtp.gmail.com" 587 "USERNAME@@gmail.com" nil)) + smtpmail-default-smtp-server "smtp.gmail.com" + smtpmail-smtp-server "smtp.gmail.com" + smtpmail-smtp-service 587) + +;; alternatively, for emacs-24 you can use: +;;(setq message-send-mail-function 'smtpmail-send-it +;; smtpmail-stream-type 'starttls +;; smtpmail-default-smtp-server "smtp.gmail.com" +;; smtpmail-smtp-server "smtp.gmail.com" +;; smtpmail-smtp-service 587) + +;; don't keep message buffers around +(setq message-kill-buffer-on-exit t) +@end lisp + +And that's it --- put the above in your emacs initialization file, change +@t{USERNAME} etc. to your own, restart Emacs, and run @kbd{M-x mu4e}. + +@node CONF Other settings +@section Other settings + +Finally, here are some more settings that are useful, but not enabled by +default for various reasons. + +@lisp +;; use 'fancy' non-ascii characters in various places in mu4e +(setq mu4e-use-fancy-chars t) + +;; save attachment to my desktop (this can also be a function) +(setq mu4e-attachment-dir "~/Desktop") + +;; attempt to show images when viewing messages +(setq mu4e-view-show-images t) +@end lisp + +@node FAQ +@appendix FAQ --- Frequently Asked Questions + +In this chapter we list a number of actual and anticipated questions and their +answers. + +@menu +* General::General questions and answers about @t{mu4e} +* Retrieving mail::Getting mail and indexing +* Reading messages::Dealing with incoming messages +* Writing messages::Dealing with outgoing messages +* Known issues::Limitations we know about +@end menu + +@node General +@section General + +@subsection Results from @t{mu} and @t{mu4e} differ - why? +@anchor{mu-mu4e-differ} In general, the same queries for @command{mu} +and @t{mu4e} should yield the same results. If they differ, this is +usually because one of the following reasons: +@itemize +@item different options: +@t{mu4e} defaults to having @t{mu4e-headers-include-related}, and +@t{mu4e-headers-results-limit} set to 500. However, the command-line +@command{mu find}'s corresponding @t{--include-related} is false, and +there's no limit (@t{--maxnum}). +@item reverse sorting: +The results may be different when @t{mu4e} and @command{mu find} do +not both sort their results in the same direction. +@item shell quoting issues: +Depending on the shell, various shell metacharacters in search query +(such as @t{*}) may be expanded by the shell before @command{mu} ever +sees them, and the query may not be what you think it is. Quoting is +necessary. +@end itemize + +@subsection The unread/all counts in the main-screen differ from the 'real' numbers - what's going on? +For speed reasons, the counts do not exclude messages that no longer exist in +the file-system, nor does it exclude duplicate messages; @xref{mu-mu4e-differ}. + +@subsection How can I quickly delete/move/trash a lot of messages? +You can select ('mark' in Emacs-speak) messages, just like you would select text +in a buffer; the actions you then take (e.g., @key{DEL} for delete, @key{m} for +move and @key{t} for trash) apply to all selected messages. You can also use +functions like @code{mu4e-headers-mark-thread} (@key{T}), +@code{mu4e-headers-mark-subthread} (@key{t}) to mark whole threads at the same +time, and @code{mu4e-headers-mark-pattern} (@key{%}) to mark all messages +matching a certain regular expression. + +@subsection Can I automatically apply the marks on messages when leaving the headers buffer? +Yes you can --- see the documentation for the variable +@t{mu4e-headers-leave-behavior}. + +@subsection How can I set @t{mu4e} as the default e-mail client in Emacs? +See @ref{Default email client}. + +@subsection Can @t{mu4e} use some fancy Unicode instead of these boring plain-ASCII ones? +Glad you asked! Yes, if you set @code{mu4e-use-fancy-chars} to @t{t}, @t{mu4e} +uses such fancy characters in a number of places. Since not all fonts include +all characters, you may want to install the @t{unifont} and/or @t{symbola} fonts +on your system. + +@subsection Can I start @t{mu4e} in the background? +Yes --- if you provide a prefix-argument (@key{C-u}), @t{mu4e} starts, but does +not show the main-window. + +@subsection Does @t{mu4e} support searching for CJK (Chinese-Japanese-Korean) characters? +Only partially. If you have @t{Xapian} 1.2.8 or newer, and set the environment +variable @t{XAPIAN_CJK_NGRAM} to non-empty before indexing, both when using +@t{mu} from the command-line and from @t{mu4e}. + +@subsection How can I customize the function to select a folder? +The @t{mu4e-completing-read-function} variable can be customized to select a +folder in any way. The variable can be set to a function that receives five +arguments, following @t{completing-read}. The default value is +@code{ido-completing-read}; to use emacs's default behavior, set the variable to +@code{completing-read}. Helm users can use the same value, and by enabling +@code{helm-mode} use helm-style completion. + +@subsection With a lot of Maildir folders, jumping to them can get slow. What can I do? +Set @code{mu4e-cache-maildir-list} to @code{t} (make sure to read its +docstring). + +@subsection How can I hide certain messages from the search results? +See the variables @code{mu4e-headers-hide-predicate} and +@code{mu4e-headers-hide-enabled}. The latter can be toggled through +@code{mu4e-headers-toggle-property}. + +For example, to filter out GMail's spam folder, set it to: +@lisp +(setq mu4e-headers-hide-predicate + (lambda (msg) + (string-suffix-p "Spam" (mu4e-message-field msg :maildir)))) +@end lisp + +@subsection I'm getting an error 'Variable binding depth exceeds max-specpdl-size' when using mu4e -- what can I do about it? +The error occurs because @t{mu4e} is binding more variables than +@t{emacs} allows for, by default. You can avoid this by setting a +higher value, e.g. by adding the following to your configuration: +@lisp +(setq max-specpdl-size 5000) +@end lisp +Note that Emacs 29 obsoletes this variable. + +@node Retrieving mail +@section Retrieving mail + +@subsection How can I get notifications when receiving mail? +There is @code{mu4e-index-updated-hook}, which gets triggered when the +indexing process triggered sees an update (not just new mail though). To +use this hook, put something like the following in your setup (assuming +you have @t{aplay} and some soundfile, change as needed): +@lisp +(add-hook 'mu4e-index-updated-hook + (defun new-mail-sound () + (shell-command "aplay ~/Sounds/boing.wav&"))) +@end lisp + +@subsection I'm getting mail through a local mailserver. What should I use for @code{mu4e-get-mail-command}? +Use the literal string @t{"true"} (or don't do anything, it's the +default) which then uses @t{/bin/true} (a command that does nothing and +always succeeds). This makes getting mail a no-op, but the messages are +still re-indexed. + +@subsection How can I re-index my messages without getting new mail? +Use @kbd{M-x mu4e-update-index} + +@subsection When I try to run @t{mu index} while @t{mu4e} is running I get errors +For instance: +@verbatim +mu: mu_store_new_writable: xapian error + 'Unable to get write lock on ~/.cache/mu/xapian: already locked +@end verbatim +What to do about this? You get this error because the underlying Xapian +database is locked by some other process; it can be opened only once in +read-write mode. There is not much @t{mu4e} can do about this, but if is +another @command{mu} instance that is holding the lock, you can ask it +to (gracefully) terminate: +@verbatim + pkill -2 -u $UID mu # send SIGINT + sleep 1 + mu index +@end verbatim +@t{mu4e} automatically restarts @t{mu} when it needs it. In practice, this +seems to work quite well. + +@subsection How can I disable the @t{Indexing...} messages? +Set the variable @code{mu4e-hide-index-messages} to non-@t{nil}. + +@subsection IMAP-synchronization and file-name changes +Some IMAP-synchronization programs such as @t{mbsync} (but not +@t{offlineimap}) don't like it when message files do not change their +names when they are moved to different folders. @t{mu4e} can attempt to +help with this - you can set the variable +@code{mu4e-change-filenames-when-moving} to non-@t{nil}. + +@subsection @command{offlineimap} and UTF-7 +@command{offlineimap} uses IMAP's UTF-7 for encoding non-ascii folder +names, while @command{mu} expects UTF-8 (so, e.g. @t{/まりもえ +お}@footnote{some Japanese characters} becomes @t{/&MH4wijCCMEgwSg-}). + +This is best solved by telling @command{offlineimap} to use UTF-8 instead --- +see @uref{https://github.com/djcb/mu/issues/68#issuecomment-8598652,this +ticket}. + +@subsection @command{mbsync} or @command{offlineimap} do not sync properly +Unfortunately, @command{mbsync} and/or @command{offlineimap} do not +always agree with @t{mu} about the meaning of various Maildir-flags. If +you encounter unexpected behavior, it is recommended you check before +and after a sync-operation. If the problem only shows up @emph{after} +sync'ing, the problem is with the sync-program, and it's most productive +to complain there. + +Also, you may want to ensure that @t{mu4e-index-lazy-check} is kept at +its default (@t{nil}) value, since it seems @command{mbsync} can make +changes that escape a 'lazy' check. + +Furthermore, there have been quite a few related queries on the +mailing-list; worthwhile to check out. + +@node Reading messages +@section Reading messages + +@subsection Opening messages is slower than expected - why? +@t{mu4e} is designed to be very fast, even with large amounts of mail. +However, if you experience slowdowns, here are some things to consider: +@itemize +@item opening messages while indexing: +@t{mu4e} communicates with the @t{mu} server mostly synchronously; this means +that you can do only one thing at a time. The one operation that potentially +does take a bit of time is indexing of mail. Indexing does happen +asynchronously, but still can slow down @t{mu} enough that users may notice. + +For some strategies to reduce that time, see the next question. +@item getting contact information can take some time: +especially when opening @t{mu4e} the first time and you have a +@emph{lot} of contacts, it can take a few seconds to process those. Note +that @t{mu4e} 1.3 and higher only get @emph{changed} contacts in +subsequent updates (after and indexing operation), so this should be +less of a concern. And you can tweak what contacts you get using +@var{mu4e-compose-complete-only-personal}, +@var{mu4e-compose-complete-only-after} and +@var{mu4e-compose-complete-max}. +@item decryption / sign verification: +encrypted / signed messages sometimes require network access, and this +may take a while; certainly if the needed servers cannot be found. +Part of this may be that influential environment variables are not set +in the emacs environment. +@end itemize + +If you still experience unexpected slowness, you can of course file a +ticket, but please be sure to mention the following: + +@itemize +@item are all messages slow or only some messages? +@item if it's only some messages, is there something specific about them? +@item in addition, please a (sufficiently censored version of) a message that is slow +@item is opening @emph{always} slow or only sometimes? When? +@end itemize + +@subsection How can I word-wrap long lines in when viewing a message? +You can toggle between wrapped and non-wrapped states using @key{w}. If you want +to do this automatically, invoke @code{visual-line-mode} in your +@code{mu4e-view-rendered-hook} (@code{mu4e-view-mode-hook} fires too early). +@subsection How can I perform custom actions on messages and attachments? +See @ref{Actions}. +@subsection How can I prevent @t{mu4e} from automatically marking messages as `read' when I read them? +Set @code{mu4e-view-auto-mark-as-read} to @code{nil}. +@subsection Does @t{mu4e} support including all related messages in a thread, like Gmail does? +Yes --- see @ref{Including related messages}. +@subsection There seems to be a lot of duplicate messages --- how can I get rid of them? +See @ref{Skipping duplicates}. +@subsection Some messages are almost unreadable in emacs --- can I view them in an external web browser? +Indeed, airlines often send messages that heavily depend on html and +are hard to digest inside emacs. Fortunately, there's an @emph{action} +(@ref{Message view actions}) defined for this. Simply add to your +configuration: +@lisp +(add-to-list 'mu4e-view-actions + '("ViewInBrowser" . mu4e-action-view-in-browser) t) +@end lisp +Now, when viewing such a difficult message, type @kbd{aV}, and the +message opens inside a web browser. You can influence the browser to +use with @code{browse-url-generic-program}. +@subsection How can I read encrypted messages that I sent? +Since you do not own the recipient's key you typically cannot read +those mails --- so the trick is to encrypt outgoing mails with your +key, too. This can be automated by adding the following snippet to +your configuration (courtesy of user @t{kpachnis}): +@lisp +(require 'epg-config) +(setq mml2015-use 'epg + epg-user-id "gpg_key_id" + mml2015-encrypt-to-self t + mml2015-sign-with-sender t) +@end lisp + +@node Writing messages +@section Writing messages + +@subsection How can I automatically set the @t{From:}-address for a reply-message? +See @ref{Compose hooks}. + +@subsection How can I dynamically determine the folders for draft/sent/trashed messages? +See @ref{Dynamic folders}. + +@subsection How can I define aliases for (groups of) e-mail addresses? +See @ref{(emacs) Mail Aliases}. + +@subsection How can I automatically add some header to an outgoing message? +See @ref{Compose hooks}. + +@subsection How can I influence the way the original message looks when replying/forwarding? +Since @code{mu4e-compose-mode} derives from @xref{(message) Top}, you can re-use +many (though not @emph{all} of its facilities. + +@subsection How can I easily include attachments in the messages I write? +You can drag-and-drop from your desktop; alternatively, you can use @ref{(emacs) +Dired}. + +@subsection How can I start a new message-thread from a reply? +Remove the @t{In-Reply-To} header, and @t{mu4e} automatically removes +the (hidden) @t{References} header as well when sending it. This makes +the message show up as a top-level message rather than as a response. + +@subsection How can I attach an existing message? +Use @code{mu4e-action-capture-message} (i.e., @kbd{a c} in the headers + view) to `capture' the to-be-attached message, then when editing the + message, use @kbd{M-x mu4e-compose-attach-captured-message}. + +@subsection How can I sign or encrypt messages? +You can do so using Emacs' MIME-support --- check the +@t{Attachments}-menu while composing a message. Also see @ref{Signing +and encrypting}. + +@subsection Address auto-completion misses some addresses +If you have set @code{mu4e-compose-complete-only-personal} to non-nil, @t{mu4e} +only completes 'personal' addresses - so you tell it about your e-mail addresses +when setting up the database (@t{mu init}); @ref{Initializing the message +store}. + +If you cannot find specific addresses you'd expect to find, inspect the +values of @var{mu4e-compose-complete-only-personal}, +@var{mu4e-compose-complete-only-after} and +@var{mu4e-compose-complete-max}. + +@subsection How can I get rid of the message buffer after sending? +@lisp +(setq message-kill-buffer-on-exit t) +@end lisp + +@subsection Sending big messages is slow and blocks emacs --- what can I do about it? + +For this, there's @uref{https://github.com/jwiegley/emacs-async,emacs-async} +(also available from the Emacs package repository); add the following snippet to +your configuration: +@lisp +(require 'smtpmail-async) +(setq + send-mail-function 'async-smtpmail-send-it + message-send-mail-function 'async-smtpmail-send-it) +@end lisp +With this, messages are sent using a background Emacs instance. + +A word of warning though, this tends to not be as reliable as sending the +message in the normal, synchronous fashion, and people have reported silent +failures, where mail sending fails for some reason without any indication of +that. + +You can check the progress of the background delivery by checking the +@t{*Messages*}-buffer, which should show something like: +@verbatim +Delivering message to "William Shakespeare" <will@example.com>... +Mark set +Saving file /home/djcb/Maildir/sent/cur/20130706-044350-darklady:2,S... +Wrote /home/djcb/Maildir/sent/cur/20130706-044350-darklady:2,S +Sending...done +@end verbatim +The first and final messages are the most important, and there may be +considerable time between them, depending on the size of the message. + +@subsection Is it possible to view headers and messages, or compose new ones, in a separate frame or window? +Yes. There is built-in support for composing messages in a new frame or window. +Either use Emacs' standard @t{compose-mail-other-frame} (@kbd{C-x 5 m}) and +@t{compose-mail-other-window} (@kbd{C-x 4 m}) if you have set up @t{mu4e} as your Emacs +e-mailer. + +Additionally, there's the variable @code{mu4e-compose-switch} (see its +docstring) which you can customize to influence how @t{mu4e} creates new +messages. + +@subsection How can I apply format=flowed to my outgoing messages? +This enables receiving clients that support this feature to reflow +paragraphs. Plain text emails with @t{Content-Type: text/plain; +format=flowed} can be reflowed (i.e. line endings removed, paragraphs +refilled) by receiving clients that support this standard. Clients +that don't support this, show them as is, which means this feature is +truly non-invasive. + +Here's an explanatory blog post which also shows why this is a desirable +feature: @url{https://mathiasbynens.be/notes/gmail-plain-text} (if you don't +have it, your mails mostly look quite bad especially on mobile devices) and +here's the @uref{https://www.ietf.org/rfc/rfc2646.txt,RFC with all the details}. + +Since version 0.9.17, @t{mu4e} sends emails with @t{format=flowed} by setting +@lisp +(setq mu4e-compose-format-flowed t) +@end lisp + +@noindent +in your Emacs init file (@file{~/.emacs} or @file{~/.emacs.d/init.el}). The +transformation of your message into the proper format is done at the time of +sending. For this to happen properly, you should write each paragraph of your +message of as a long line (i.e. without carriage return). If you introduce +unwanted newlines in your paragraph, use @kbd{M-q} to reformat it as a single +line. + +If you want to send the message with paragraphs on single lines but +without @t{format=flowed} (because, say, the receiver does not +understand the latter as it is the case for Google or Github), use +@kbd{M-x use-hard-newlines} (to turn @code{use-hard-newlines} off) or +uncheck the box @t{format=flowed} in the @t{Text} menu when composing a +message. + +@subsection How can I force images to be shown at the end of my messages, regardless of where I insert them? +User Marcin Borkowski has a solution: +@lisp +(defun mml-attach-file--go-to-eob (orig-fun &rest args) + "Go to the end of buffer before attaching files." + (save-excursion + (save-restriction + (widen) + (goto-char (point-max)) + (apply orig-fun args)))) + +(advice-add 'mml-attach-file :around #'mml-attach-file--go-to-eob) +@end lisp + +@subsection How can I avoid Outlook display issues? + +Limited testing shows that certain Outlook clients do not work well with inline +replies, and the entire message including-and-below the first quoted section is +collapsed. This means recipients may not even notice important inline text, +especially if there is some top-posted content. This has been observed on OS X, +Windows, and Web-based Outlook clients accessing Office 365. + +It appears the bug is triggered by the standard reply regex "On ... +wrote:". Changing "On", or removing the trailing ":" appears to fix the +bug (in limited testing). Therefore, a simple work-around is to set +`message-citation-line-format` to something slightly non-standard, such +as: +@lisp +(setq message-citation-line-format "On %Y-%m-%d at %R %Z, %f wrote...") +@end lisp + +@node Known issues +@section Known issues + +Although they are not really @emph{questions}, we end this chapter with a list +of known issues and/or missing features in @t{mu4e}. Thus, users won't have to +search in vain for things that are not there (yet), and the author can use it as +a todo-list. + +@subsection UTF-8 language environment is required +@t{mu4e} does not work well if the Emacs language environment is not UTF-8; so, +if you encounter problems with encodings, be sure to have +@code{(set-language-environment "UTF-8")} in your @file{~/.emacs} (or its moral +equivalents in other places). + +@subsection Headers-buffer can get mis-aligned +Due to the way the headers buffer works, it can get misaligned. + +For the particular case where the header values are misaligned with the column +headings, you can try something like the following: +@lisp +(add-hook 'mu4e-headers-mode-hook #'my-mu4e-headers-mode-hook) +(defun my-mu4e-headers-mode-hook () + ;; Account for the fringe and other spacing in the header line. + (header-line-indent-mode 1) + (push (propertize " " 'display '(space :align-to header-line-indent-width)) + header-line-format) + ;; Ensure `text-scale-adjust' scales the header line with the headers themselves + ;; by ensuring the `default' face is in the inheritance hierarchy. + (face-remap-add-relative 'header-line '(:inherit (mu4e-header-face default))) +@end lisp + +This does not solve all possible issues; that would require a thorough rework of +the headers-view, which may happen at some time. + +@node Tips and Tricks +@appendix Tips and Tricks + +@menu +* Fancy characters:: Non-ascii characters in the UI +* Refiling messages:: Moving message to some archive folder +* Saving outgoing messages:: Automatically save sent messages +* Confirmation before sending:: Check messages before sending +@end menu + +@node Fancy characters +@section Fancy characters + +When using `fancy characters' (@code{mu4e-use-fancy-chars}) with the +@emph{Inconsolata}-font (and likely others as well), the display may be +slightly off; the reason for this issue is that Inconsolata does not +contain the glyphs for the `fancy' arrows and the glyphs that are used +as replacements are too high. + +To fix this, you can use something like the following workaround (in +your @t{.emacs}-file): +@lisp +(when (equal window-system 'x) + (set-fontset-font "fontset-default" 'unicode "Dejavu Sans Mono") + (set-face-font 'default "Inconsolata-10")) +@end lisp + +Other fonts with good support for Unicode are @t{unifont} and +@t{symbola}. + +For a more complete solution, but with greater overhead, you can also +try the @emph{unicode-fonts} package: +@lisp +(require 'unicode-fonts) +(require 'persistent-soft) ; To cache the fonts and reduce load time +(unicode-fonts-setup) +@end lisp + +It's possible to customize various header marks as well, with a ``fancy'' and +``non-fancy'' version (if you cannot see some the ``fancy'' characters, that is +an indication that the font you are using does not support those characters. + +@lisp + (setq + mu4e-headers-draft-mark '("D" . "💈") + mu4e-headers-flagged-mark '("F" . "📍") + mu4e-headers-new-mark '("N" . "🔥") + mu4e-headers-passed-mark '("P" . "❯") + mu4e-headers-replied-mark '("R" . "❮") + mu4e-headers-seen-mark '("S" . "☑") + mu4e-headers-trashed-mark '("T" . "💀") + mu4e-headers-attach-mark '("a" . "📎") + mu4e-headers-encrypted-mark '("x" . "🔒") + mu4e-headers-signed-mark '("s" . "🔑") + mu4e-headers-unread-mark '("u" . "⎕") + mu4e-headers-list-mark '("l" . "🔈") + mu4e-headers-personal-mark '("p" . "👨") + mu4e-headers-calendar-mark '("c" . "📅")) +@end lisp + +@node Refiling messages +@section Refiling messages + +By setting @code{mu4e-refile-folder} to a function, you can dynamically +determine where messages are to be refiled. If you want to do this based +on the subject of a message, you can use a function that matches the +subject against a list of regexes in the following way. First, set up a +variable @code{my-mu4e-subject-alist} containing regexes plus associated +mail folders: + +@lisp +(defvar my-mu4e-subject-alist '(("kolloqui\\(um\\|a\\)" . "/Kolloquium") + ("Calls" . "/Calls") + ("Lehr" . "/Lehre") + ("webseite\\|homepage\\|website" . "/Webseite")) + "List of subjects and their respective refile folders.") +@end lisp + +Now you can use the following function to automatically refile messages +based on their subject line: + +@lisp +(defun my-mu4e-refile-folder-function (msg) + "Set the refile folder for MSG." + (let ((subject (mu4e-message-field msg :subject)) + (folder (or (cdar (member* subject my-mu4e-subject-alist + :test #'(lambda (x y) + (string-match (car y) x)))) + "/General"))) + folder)) +@end lisp + +Note the @t{"/General"} folder: it is the default folder in case the +subject does not match any of the regexes in +@code{my-mu4e-subject-alist}. + +In order to make this work, you'll of course need to set +@code{mu4e-refile-folder} to this function: + +@lisp +(setq mu4e-refile-folder 'my-mu4e-refile-folder-function) +@end lisp + +If you have multiple accounts, you can accommodate them as well: + +@lisp +(defun my-mu4e-refile-folder-function (msg) + "Set the refile folder for MSG." + (let ((maildir (mu4e-message-field msg :maildir)) + (subject (mu4e-message-field msg :subject)) + folder) + (cond + ((string-match "Account1" maildir) + (setq folder (or (catch 'found + (dolist (mailing-list my-mu4e-mailing-lists) + (if (mu4e-message-contact-field-matches + msg :to (car mailing-list)) + (throw 'found (cdr mailing-list))))) + "/Account1/General"))) + ((string-match "Gmail" maildir) + (setq folder "/Gmail/All Mail")) + ((string-match "Account2" maildir) + (setq folder (or (cdar (member* subject my-mu4e-subject-alist + :test #'(lambda (x y) + (string-match + (car y) x)))) + "/Account2/General")))) + folder)) +@end lisp + +This function actually uses different methods to determine the refile +folder, depending on the account: for @emph{Account2}, it uses +@code{my-mu4e-subject-alist}, for the @emph{Gmail} account it simply uses the +folder ``All Mail''. For Account1, it uses another method: it files the +message based on the mailing list to which it was sent. This requires +another variable: + +@lisp +(defvar my-mu4e-mailing-lists + '(("mu-discuss@@googlegroups.com" . "/Account1/mu4e") + ("pandoc-discuss@@googlegroups.com" . "/Account1/Pandoc") + ("auctex@@gnu.org" . "/Account1/AUCTeX")) + "List of mailing list addresses and folders where + their messages are saved.") +@end lisp + +@node Saving outgoing messages +@section Saving outgoing messages + +Like @code{mu4e-refile-folder}, the variable @code{mu4e-sent-folder} can +also be set to a function, in order to dynamically determine the save +folder. One might, for example, wish to automatically put messages going +to mailing lists into the trash (because you'll receive them back from +the list anyway). If you have set up the variable +@code{my-mu4e-mailing-lists} as mentioned, you can use the following +function to determine a 'sent'-folder: + +@lisp +(defun my-mu4e-sent-folder-function (msg) + "Set the sent folder for the current message." + (let ((from-address (message-field-value "From")) + (to-address (message-field-value "To"))) + (cond + ((string-match "my.address@@account1.example.com" from-address) + (if (member* to-address my-mu4e-mailing-lists + :test #'(lambda (x y) + (string-match (car y) x))) + "/Trash" + "/Account1/Sent")) + ((string-match "my.address@@gmail.com" from-address) + "/Gmail/Sent Mail") + (t (mu4e-ask-maildir-check-exists "Save message to maildir: "))))) +@end lisp + +Note that this function doesn't use @code{(mu4e-message-field msg +:maildir)} to determine which account the message is being sent from. +The reason is that the function in @code{mu4e-sent-folder} is +called when you send the message, but before @t{mu4e} has created the +message struct from the compose buffer, so that +@code{mu4e-message-field} cannot be used. Instead, the function uses +@code{message-field-value}, which extracts the values of the headers in +the compose buffer. This means that it is not possible to extract the +account name from the message's maildir, so instead the from address is +used to determine the account. + +Again, the function shows three different possibilities: for the first +account (@t{my.address@@account1.example.com}) it uses +@code{my-mu4e-mailing-lists} again to determine if the message goes to a +mailing list. If so, the message is put in the trash folder, if not, it +is saved in @t{/Account1/Sent}. For the second (Gmail) account, sent +mail is simply saved in the Sent Mail folder. + +If the from address is not associated with Account1 or with the Gmail +account, the function uses @code{mu4e-ask-maildir-check-exists} to ask +the user for a maildir to save the message in. + +@node Confirmation before sending +@section Confirmation before sending + +To protect yourself from sending messages too hastily, you can add a +final confirmation, which you can of course make as elaborate as you +wish. + +@lisp +(defun confirm-empty-subject () + "Require confirmation before sending without subject." + (let ((sub (message-field-value "Subject"))) + (or (and sub (not (string-match "\\`[ \t]*\\'" sub))) + (yes-or-no-p "Really send without Subject? ") + (keyboard-quit)))) + +(add-hook 'message-send-hook #'confirm-empty-subject) +@end lisp + +If you @emph{always} want to be asked for for confirmation, set +@code{message-confirm-send} to non-@t{nil} so the question ``Send message?'' is +asked for confirmation. + +@node How it works +@appendix How it works + +While perhaps not interesting for all users of @t{mu4e}, some curious +souls may want to know how @t{mu4e} does its job. + +@menu +* High-level overview::How the pieces fit together +* mu server::The mu process running in the background +* Reading from the server::Processing responses from the server +* The message s-expression::What messages look like from the inside +@end menu + +@node High-level overview +@section High-level overview + +At a high level, we can summarize the structure of the @t{mu4e} system using +some ascii-art: + +@cartouche +@example + +---------+ + | emacs | + | +------+ + +----| mu4e | --> send mail (smtpmail) + +------+ + | A + V | ---/ search, view, move mail + +---------+ \ + | mu | + +---------+ + | A + V | + +---------+ + | Maildir | <--- receive mail (fetchmail, + +---------+ offlineimap, ...) +@end example +@end cartouche + +In words: +@itemize +@item Your e-mail messages are stored in a Maildir-directory +(typically, @file{~/Maildir} and its subdirectories), and new mail comes in +using tools like @t{fetchmail}, @t{offlineimap}, or through a local mail +server. +@item @t{mu} indexes these messages periodically, so you can quickly search for +them. @t{mu} can run in a special @t{server}-mode, where it provides services + to client software. +@item @t{mu4e}, which runs inside Emacs is + such a client; it communicates with @command{mu} (in its @t{server}-mode) to + search for messages, and manipulate them. +@item @t{mu4e} uses the facilities + offered by Emacs (the Gnus message editor and @t{smtpmail}) to send + messages. +@end itemize + +@node mu server +@section @t{mu server} + +@t{mu4e} is based on the @t{mu} e-mail searching/indexer. The latter +is a C++-program; there are different ways to communicate with a +client that is emacs-based. + +One way to implement this, would be to call the @t{mu} command-line +tool with some parameters and then parse the output. In fact, that was +the first approach --- @t{mu4e} would invoke e.g., @t{mu find} and +process the output in Emacs. + +However, with this approach, we need to load the entire e-mail +@emph{Xapian} database (in which the message is stored) for each +invocation. Wouldn't it be nicer to keep a running @t{mu} instance +around? Indeed, it would --- and thus, the @t{mu server} sub-command +was born. Running @t{mu server} starts a simple shell, in which you +can give commands to @command{mu}, which then spits out the +results/errors. @command{mu server} is not meant for humans, but it +can be used manually, which is great for debugging. + +@node Reading from the server +@section Reading from the server + +In the design, the next question was what format @t{mu} should use for its +output for @t{mu4e} (Emacs) to process. Some other programs use +@abbr{JSON} here, but it seemed easier (and possibly, more efficient) just to +talk to Emacs in its native language: @emph{s-expressions}, and +interpret those using the Emacs-function +@code{read-from-string}. See @ref{The message s-expression} for details on the +format. + +So, now let's look at how we process the data from @t{mu server} in +Emacs. We'll leave out a lot of details, @t{mu4e}-specifics, and look +at a bit more generic approach. + +The first thing to do is to create a process (for example, with +@code{start-process}), and then register a filter function for it, which is +invoked whenever the process has some data for us. Something like: + +@lisp + (let ((proc (start-process <arguments>))) + (set-process-filter proc 'my-process-filter) + (set-process-sentinel proc 'my-process-sentinel)) +@end lisp + +Note, the process sentinel is invoked when the process is terminated +--- so there you can clean things up. The function +@code{my-process-filter} is a user-defined function that takes the +process and the chunk of output as arguments; in @t{mu4e} it looks +something like (pseudo-lisp): + +@lisp +(defun my-process-filter (proc str) + ;; mu4e-buf: a global string variable to which data gets appended + ;; as we receive it + (setq mu4e-buf (concat mu4e-buf str)) + (when <we-have-received-a-full-expression> + <eat-expression-from mu4e-buf> + <evaluate-expression>)) +@end lisp + +@code{<evaluate-expression>} de-multiplexes the s-expression we got. +For example, if the s-expression looks like an e-mail message header, +it is processed by the header-handling function, which appends it to +the header list. If the s-expression looks like an error message, it +is reported to the user. And so on. + +The language between frontend and backend is documented partly in the +@t{mu-server} man-page and more completely in the output of @t{mu +server --commands}. + +@t{mu4e} can log these communications; you can use @kbd{M-x +mu4e-toggle-logging} to turn logging on and off, and you can view the +log using @kbd{M-x mu4e-show-log} (@key{$}). + +@node The message s-expression +@section The message s-expression + +As a word of warning, the details of the s-expression are internal to the mu4e - +mu communications, and are subject to change between versions. + +A typical message s-expression looks something like the following: + +@lisp +(:docid 32461 + :from ((:name "Nikola Tesla" :email "niko@@example.com")) + :to ((:name "Thomas Edison" :email "tom@@example.com")) + :cc ((:name "Rupert The Monkey" :email "rupert@@example.com")) + :subject "RE: what about the 50K?" + :date (20369 17624 0) + :size 4337 + :message-id "C8233AB82D81EE81AF0114E4E74@@123213.mail.example.com" + :path "/home/tom/Maildir/INBOX/cur/133443243973_1.10027.atlas:2,S" + :maildir "/INBOX" + :priority normal + :flags (seen attach) + .... +") +@end lisp + +This s-expression forms a property list (@t{plist}), and we can get +values from it using @t{plist-get}; for example @code{(plist-get msg +:subject)} would get you the message subject. However, it's better to +use the function @code{mu4e-message-field} to shield you from some of +the implementation details that are subject to change; and see the other +convenience functions in @file{mu4e-message.el}. + +Some notes on the format: +@itemize +@item The address fields are @emph{lists} of @t{plists} of the form +@code{(:name <name> :email <email>)}, where @t{name} can be @t{nil}. +@item The date is in format Emacs uses (for example in +@code{current-time}).@footnote{Emacs 32-bit integers have only 29 bits +available for the actual number; the other bits are use by Emacs for +internal purposes. Therefore, we need to split @t{time_t} in two +numbers.} +@end itemize + +@subsection Example: ping-pong + +As an example of the communication between @t{mu4e} and @command{mu}, +let's look at the @t{ping-pong}-sequence. When @t{mu4e} starts, it +sends a command @t{ping} to the @t{mu server} backend, to learn about +its version. @t{mu server} then responds with a @t{pong} s-expression +to provide this information (this is implemented in +@file{mu-cmd-server.c}). + +We start this sequence when @t{mu4e} is invoked (when the program is +started). It calls @t{mu4e--server-ping}, and registers a (lambda) +function for @t{mu4e-server-pong-func}, to handle the response. + +@verbatim +-> (ping) +<-<prefix>(:pong "mu" :props (:version "x.x.x" :doccount 78545)) +@end verbatim + +When we receive such a @t{pong} (in @file{mu4e-server.el}), the lambda +function we registered is called, and it compares the version we got +from the @t{pong} with the version we expected, and raises an error if +they differ. + +@node Debugging +@appendix Debugging + +As explained in @ref{How it works}, @t{mu4e} communicates with its +backend (@t{mu server}) by sending commands and receiving responses +(s-expressions). + +For debugging purposes, it can be very useful to see this data. For +this reason, @t{mu4e} can log all these messages. Note that the +`protocol' is documented to some extent in the @t{mu-server} manpage. + +You can enable (and disable) logging with @kbd{M-x +mu4e-toggle-logging}. The log-buffer is called @t{*mu4e-log*}, and in +the @ref{Main view}, @ref{Headers view} and @ref{Message view}, +there's a keybinding @key{$} that takes you there. You can quit it by +pressing @key{q}. + +Logging can be a bit resource-intensive, so you may not want to leave +it on all the time. By default, the log only maintains the most recent +1200 lines. @t{mu} itself keeps a log as well, you can find it in +@t{<MUHOME>/mu.log}, on Unix typically @t{~/.cache/mu/mu.log}. + +@node GNU Free Documentation License +@appendix GNU Free Documentation License + +@include fdl.texi + +@c @node Command Index +@c @unnumbered Command and Function Index +@c @printindex fn + +@c @node Variable Index +@c @unnumbered Variable Index +@c @printindex vr + +@node Concept Index +@unnumbered Concept Index +@printindex cp + +@bye + +@c Local Variables: +@c coding: utf-8 +@c End: diff --git a/mu4e/texinfo-klare.css b/mu4e/texinfo-klare.css new file mode 100644 index 0000000..e54a882 --- /dev/null +++ b/mu4e/texinfo-klare.css @@ -0,0 +1,228 @@ +/* + Custom CSS for HTML documents generated with Texinfo's makeinfo. + Public domain 2016 sirgazil. All rights waived. +*/ + + + +/* NATIVE ELEMENTS */ +a:link, +a:visited { + color: #245C8A; + text-decoration: none; +} + +a:active, +a:focus, +a:hover { + text-decoration: underline; +} + +abbr, +acronym { + cursor: help; +} + +blockquote { + color: #555753; + font-style: oblique; + margin: 30px 0px; + padding-left: 3em; +} + +body { + background-color: white; + box-shadow: 0 0 2px gray; + box-sizing: border-box; + color: #333; + font-family: sans-serif; + font-size: 16px; + margin: 50px auto; + max-width: 960px; + padding: 50px; +} + +code, +samp, +tt, +var { + color: purple; + font-size: 0.8em; +} + +div.example, +div.lisp { + margin: 0px; +} + +dl { + margin: 3em 0em; +} + +dl dl { + margin: 0em; +} + +dt { + background-color: #F5F5F5; + padding: 0.5em; +} + +h1, +h2, +h2.contents-heading, +h3, +h4 { + padding: 20px 0px 0px 0px; + font-weight: normal; +} + +h1 { + font-size: 2.4em; +} + +h2 { + font-size: 2.2em; + font-weight: bold; +} + +h3 { + font-size: 1.8em; +} + +h4 { + font-size: 1.4em; +} + +hr { + background-color: silver; + border-style: none; + height: 1px; + margin: 0px; +} + +html { + background-color: #F5F5F5; +} + +img { + max-width: 100%; +} + +li { + padding: 5px; +} + +pre.display, +pre.example, +pre.format, +pre.lisp, +pre.verbatim{ + overflow: auto; +} + +pre.example, +pre.lisp, +pre.verbatim { + background-color: #2D3743; + border-color: #000; + border-style: solid; + border-width: thin; + color: #E1E1E1; + font-size: smaller; + padding: 1em; +} + +pre.menu-comment { + border-color: #E4E4E4; + border-bottom-style: solid; + border-width: thin; + font-family: sans; +} + +table { + border-collapse: collapse; + margin: 40px 0px; +} + +table.index-cp *, +table.index-fn *, +table.index-ky *, +table.index-pg *, +table.index-tp *, +table.index-vr * { + background-color: inherit; + border-style: none; +} + +td, +th { + border-color: silver; + border-style: solid; + border-width: thin; + padding: 10px; +} + +th { + background-color: #F5F5F5; +} +/* END NATIVE ELEMENTS */ + + + +/* CLASSES */ +.contents { + margin-bottom: 4em; +} + +.float { + margin: 3em 0em; +} + +.float-caption { + font-size: smaller; + text-align: center; +} + +.float > img { + display: block; + margin: auto; +} + +.footnote { + font-size: smaller; + margin: 5em 0em; +} + +.footnote h3 { + display: inline; + font-size: small; +} + +.header { + background-color: #F2F2F2; + font-size: small; + padding: 0.2em 1em; +} + +.key { + color: purple; + font-size: 0.8em; +} + +.menu * { + border-style: none; +} + +.menu td { + padding: 0.5em 0em; +} + +.menu td:last-child { + width: 60%; +} + +.menu th { + background-color: inherit; +} +/* END CLASSES */ diff --git a/testdata/cjk/cur/test1 b/testdata/cjk/cur/test1 new file mode 100644 index 0000000..1538790 --- /dev/null +++ b/testdata/cjk/cur/test1 @@ -0,0 +1,10 @@ +From: "Bob" <bob@builder.com> +Subject: CJK 1 +To: "Chase" <chase@ppatrol.org> +Date: Thu, 18 Nov 2021 08:35:34 +0200 +Message-Id: 112342343e9dfo.fsf@builder.com +User-Agent: mu4e 1.7.5; emacs 29.0.50 + + サーバがダウンしました + +https://github.com/djcb/mu/issues/1428 diff --git a/testdata/cjk/cur/test2 b/testdata/cjk/cur/test2 new file mode 100644 index 0000000..875bff5 --- /dev/null +++ b/testdata/cjk/cur/test2 @@ -0,0 +1,10 @@ +From: "Bob" <bob@builder.com> +Subject: CJK 2 +To: "Chase" <chase@ppatrol.org> +Date: Thu, 18 Nov 2021 08:35:34 +0200 +Message-Id: 271r2342343e9dfo.fsf@builder.com +User-Agent: mu4e 1.7.5; emacs 29.0.50 + + スポンサーシップ募集 + +https://github.com/djcb/mu/issues/1428 diff --git a/testdata/cjk/cur/test3 b/testdata/cjk/cur/test3 new file mode 100644 index 0000000..f0efe71 --- /dev/null +++ b/testdata/cjk/cur/test3 @@ -0,0 +1,10 @@ +From: "Bob" <bob@builder.com> +Subject: CJK 3 +To: "Chase" <chase@ppatrol.org> +Date: Thu, 18 Nov 2021 08:35:34 +0200 +Message-Id: 3871r2342343e9dfo.fsf@builder.com +User-Agent: mu4e 1.7.5; emacs 29.0.50 + + サービス開始について + +https://github.com/djcb/mu/issues/1428 diff --git a/testdata/cjk/cur/test4 b/testdata/cjk/cur/test4 new file mode 100644 index 0000000..2bad399 --- /dev/null +++ b/testdata/cjk/cur/test4 @@ -0,0 +1,10 @@ +From: "Bob" <bob@builder.com> +Subject: CJK 4 +To: "Chase" <chase@ppatrol.org> +Date: Thu, 18 Nov 2021 08:35:34 +0200 +Message-Id: 4871r2342343e9dfo.fsf@builder.com +User-Agent: mu4e 1.7.5; emacs 29.0.50 + + ショルダーバック + +https://github.com/djcb/mu/issues/1428 diff --git a/testdata/testdir/cur/1220863042.12663_1.mindcrime!2,S b/testdata/testdir/cur/1220863042.12663_1.mindcrime!2,S new file mode 100644 index 0000000..ab1500f --- /dev/null +++ b/testdata/testdir/cur/1220863042.12663_1.mindcrime!2,S @@ -0,0 +1,146 @@ +Return-Path: <gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-4.9 required=3.0 tests=BAYES_00,DATE_IN_PAST_96_XX, + RCVD_IN_DNSWL_MED autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 5123469CB3 + for <xxxx@localhost>; Thu, 7 Aug 2008 08:10:19 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Thu, 07 Aug 2008 08:10:19 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs39272wfh; Wed, 6 Aug 2008 + 20:15:17 -0700 (PDT) +Received: by 10.65.133.8 with SMTP id k8mr2071878qbn.7.1218078916289; Wed, 06 + Aug 2008 20:15:16 -0700 (PDT) +Received: from sourceware.org (sourceware.org [209.132.176.174]) by + mx.google.com with SMTP id 28si7904461qbw.0.2008.08.06.20.15.15; Wed, 06 Aug + 2008 20:15:16 -0700 (PDT) +Received-SPF: neutral (google.com: 209.132.176.174 is neither permitted nor + denied by domain of gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org) + client-ip=209.132.176.174; +Authentication-Results: mx.google.com; spf=neutral (google.com: + 209.132.176.174 is neither permitted nor denied by domain of + gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org) + smtp.mail=gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org +Received: (qmail 13493 invoked by alias); 7 Aug 2008 03:15:13 -0000 +Received: (qmail 13485 invoked by uid 22791); 7 Aug 2008 03:15:12 -0000 +Received: from mailgw1a.lmco.com (HELO mailgw1a.lmco.com) (192.31.106.7) + by sourceware.org (qpsmtpd/0.31) with ESMTP; Thu, 07 Aug 2008 03:14:27 +0000 +Received: from emss07g01.ems.lmco.com (relay5.ems.lmco.com [166.29.2.16])by + mailgw1a.lmco.com (LM-6) with ESMTP id m773EPZH014730for + <gcc-help@gcc.gnu.org>; Wed, 6 Aug 2008 21:14:25 -0600 (MDT) +Received: from CONVERSION2-DAEMON.lmco.com by lmco.com (PMDF V6.3-x14 #31428) + id <0K5700601NO18J@lmco.com> for gcc-help@gcc.gnu.org; Wed, 06 Aug 2008 + 21:14:25 -0600 (MDT) +Received: from EMSS04I00.us.lmco.com ([166.17.13.135]) by lmco.com (PMDF + V6.3-x14 #31428) with ESMTP id <0K5700H5MNNWGX@lmco.com> for + gcc-help@gcc.gnu.org; Wed, 06 Aug 2008 21:14:20 -0600 (MDT) +Received: from EMSS35M06.us.lmco.com ([158.187.107.143]) by + EMSS04I00.us.lmco.com with Microsoft SMTPSVC(5.0.2195.6713); Wed, 06 Aug + 2008 23:14:20 -0400 +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "Mickey Mouse" <anon@example.com> +Subject: gcc include search order +To: "Donald Duck" <gcc-help@gcc.gnu.org> +Message-id: <3BE9E6535E3029448670913581E7A1A20D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Content-class: urn:content-classes:message +Mailing-List: contact gcc-help-help@gcc.gnu.org; run by ezmlm +Precedence: klub +List-Id: <gcc-help.gcc.gnu.org> +List-Unsubscribe: <mailto:gcc-help-unsubscribe-xxxx.klub=gmail.com@gcc.gnu.org> +List-Archive: <http://gcc.gnu.org/ml/gcc-help/> +List-Post: <mailto:gcc-help@gcc.gnu.org> +List-Help: <mailto:gcc-help-help@gcc.gnu.org> +Sender: gcc-help-owner@gcc.gnu.org +Delivered-To: mailing list gcc-help@gcc.gnu.org +Content-Length: 3024 + + +Hi. +In my unit testing I need to change some header files (target is +vxWorks, which supports some things that the sun does not). +So, what I do is fetch the development tree, and then in a new unit test +directory I attempt to compile the unit under test. Since this is NOT +vxworks, I use sed to change some of the .h files and put them in a +./changed directory. + +When I try to compile the file, it is still using the .h file from the +original location, even though I have listed the include path for +./changed before the include path for the development tree. + +Here is a partial output from gcc using the -v option + +GNU CPP version 3.1 (cpplib) (sparc ELF) +GNU C++ version 3.1 (sparc-sun-solaris2.8) + compiled by GNU C version 3.1. +ignoring nonexistent directory "NONE/include" +#include "..." search starts here: +#include <...> search starts here: + . + changed + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/mp/interface + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/ap/app + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/interface + /usr/local/include/g++-v3 + /usr/local/include/g++-v3/sparc-sun-solaris2.8 + /usr/local/include/g++-v3/backward + /usr/local/include + /usr/local/lib/gcc-lib/sparc-sun-solaris2.8/3.1/include + /usr/local/sparc-sun-solaris2.8/include + /usr/include +End of search list. + +I know the changed file is correct and that the include is not working +as expected, because when I copy the file from ./changed, back into the +development tree, the compilation works as expected. + +One more bit of information. The source that I cam compiling is in +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/ap/app +And it is including files from +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common +These include files should be including the files from ./changed (when +they exist) but they are ignoring the .h files in the ./changed +directory and are instead using other, unchanged files in the +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common +directory. + +The gcc command line is something like + + TEST_DIR="." + + CHANGED_DIR_NAME=changed + CHANGED_FILES_DIR=${TEST_DIR}/${CHANGED_DIR_NAME} + + CICU_HEADER_FILES="-I ${AP_INTERFACE_FILES} -I ${AP_APP_FILES} -I +${SHARED_COMMON_FILES} -I ${SHARED_INTERFACE_FILES}" + + HEADERS="-I ./ -I ${CHANGED_FILES_DIR} ${CICU_HEADER_FILES}" + DEFINES="-DSUNRUN -DA10_DEBUG -DJOETEST" + + CFLAGS="-v -c -g -O1 -pipe -Wformat -Wunused -Wuninitialized -Wshadow +-Wmissing-prototypes -Wmissing-declarations" + + printf "Compiling the UUT File\n" + gcc -fprofile-arcs -ftest-coverage ${CFLAGS} ${HEADERS} ${DEFINES} +${AP_APP_FILES}/unitUnderTest.cpp + + +I hope this explanation is clear. If anyone knows how to fix the command +line so that it gets the .h files in the "changed" directory are used +instead of files in the other include directories. + +Thanks +Joe + +---------------------------------------------------- +Time Flies like an Arrow. Fruit Flies like a Banana + + diff --git a/testdata/testdir/cur/1220863060.12663_3.mindcrime!2,S b/testdata/testdir/cur/1220863060.12663_3.mindcrime!2,S new file mode 100644 index 0000000..d0ff0d7 --- /dev/null +++ b/testdata/testdir/cur/1220863060.12663_3.mindcrime!2,S @@ -0,0 +1,230 @@ +Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00,HTML_MESSAGE + autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id D724F6963B + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:27 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Mon, 04 Aug 2008 21:49:27 +0300 (EEST) +Received: by 10.142.51.12 with SMTP id y12cs86537wfy; Mon, 4 Aug 2008 00:38:51 + -0700 (PDT) +Received: by 10.151.113.5 with SMTP id q5mr272266ybm.37.1217835529913; Mon, 04 + Aug 2008 00:38:49 -0700 (PDT) +Received: from sqlite.org (sqlite.org [67.18.92.124]) by mx.google.com with + ESMTP id 5si5754915ywd.8.2008.08.04.00.38.30; Mon, 04 Aug 2008 00:38:50 -0700 + (PDT) +Received-SPF: pass (google.com: best guess record for domain of + sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as permitted sender) + client-ip=67.18.92.124; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record + for domain of sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as + permitted sender) smtp.mail=sqlite-dev-bounces@sqlite.org +Received: from sqlite.org (localhost [127.0.0.1]) by sqlite.org (Postfix) with + ESMTP id 765A511C46; Mon, 4 Aug 2008 03:38:27 -0400 (EDT) +X-Original-To: sqlite-dev@sqlite.org +Delivered-To: sqlite-dev@sqlite.org +Received: from ik-out-1112.google.com (ik-out-1112.google.com [66.249.90.176]) + by sqlite.org (Postfix) with ESMTP id 4C59511C41 for <sqlite-dev@sqlite.org>; + Mon, 4 Aug 2008 03:38:23 -0400 (EDT) +Received: by ik-out-1112.google.com with SMTP id b32so2163423ika.0 for + <sqlite-dev@sqlite.org>; Mon, 04 Aug 2008 00:38:23 -0700 (PDT) +Received: by 10.210.54.19 with SMTP id c19mr14589042eba.107.1217835502549; + Mon, 04 Aug 2008 00:38:22 -0700 (PDT) +Received: by 10.210.115.10 with HTTP; Mon, 4 Aug 2008 00:38:22 -0700 (PDT) +Message-ID: <477821040808040038s381bf382p7411451e3c1a2e4e@mail.gmail.com> +Date: Mon, 4 Aug 2008 10:38:22 +0300 +From: anon@example.com +To: sqlite-dev@sqlite.org +In-Reply-To: <73d4fc50808030747g303a170ieac567723c2d4f24@mail.gmail.com> +MIME-Version: 1.0 +References: <477821040808030533y41f1501dq32447b568b6e6ca5@mail.gmail.com> + <73d4fc50808030747g303a170ieac567723c2d4f24@mail.gmail.com> +Subject: Re: [sqlite-dev] SQLite exception A&B +X-BeenThere: sqlite-dev@sqlite.org +X-Mailman-Version: 2.1.9 +Priority: normal +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +List-Unsubscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=unsubscribe> +List-Archive: <http://sqlite.org:8080/cgi-bin/mailman/private/sqlite-dev> +List-Post: <mailto:sqlite-dev@sqlite.org> +List-Help: <mailto:sqlite-dev-request@sqlite.org?subject=help> +List-Subscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=subscribe> +Content-Type: multipart/mixed; boundary="===============2123623832==" +Mime-version: 1.0 +Sender: sqlite-dev-bounces@sqlite.org +Errors-To: sqlite-dev-bounces@sqlite.org +Content-Length: 8475 + +--===============2123623832== +Content-Type: multipart/alternative; + boundary="----=_Part_29556_25702991.1217835502493" + +------=_Part_29556_25702991.1217835502493 +Content-Type: text/plain; charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +Hi Grant, + +Thanks for your reply. +I am using a different session for each thread, whenever a thread wishes to +access the database it gets a session from the session pool and works with +that session until its work is done. + +Most of the actions the threads are doing on the database are quite +complicated and are required to be fully committed or completely ignored, so +yes, I am (most of the time) explicitly beginning and committing my +transactions. + +Regarding the SQLiteStatementImpl, I believe the Poco manual explains that +sessions and statements for that matter cannot be shared between threads, +therefore if you are using a session via one thread only it should work +fine. + +My first impression was that the problem was in the Poco infrastructure (I +have found several Poco related bugs in the past), but the problem ALWAYS +occurs when I perform the "BEGIN IMMEDIATE" action, if it were a Poco +related bug, I would expect to see it here and there without any relation to +this specific statement, but that is not the case. + +None the less, I will also post my question on the Poco forums. + +Nadav. + +On Sun, Aug 3, 2008 at 5:47 PM, Grant Gatchel <grant.gatchel@gmail.com>wrote: + +> Are you using the same Poco::Session for every thread or does each call +> create a new session/handle to the database? +> +> Are you explicitly BEGINning and COMMITting your transactions? +> +> In looking at the 1.3.2 branch of Poco::Data::SQLite, there appears to be a +> race condition in the SQLiteStatementImpl::next() method in which the member +> _nextResponse is being accessed before the SQLiteStatementImpl::hasNext() +> method has a chance to interpret that value and throw an exception. +> +> This question might be more suitable in the Poco forums or mailinglist. +> +> - Grant +> +> On Sun, Aug 3, 2008 at 8:33 AM, nadav g <nadav.gr@gmail.com> wrote: +> +>> Hi All, +>> +>> I have been using SQLite with Poco (www.appinf.com) as my infrastructure. +>> The program is running several threads that access this database very +>> often and are synchronized by SQLite itself. +>> Everything seems to work just fine most of time (usually days - weeks) but +>> I do get an occasional exception: +>> +>> Exception: SQL error or missing database: Iterator Error: trying to check +>> if there is a next value +>> +>> The backtrace leads to this statement: +>> *"BEGIN IMMEDIATE"* +>> +>> This specific code runs numerous times before an exception occurs (if +>> occurs at all) and I cannot think of any reason for it to fail later rather +>> than sooner. +>> It is pretty obvious that this situation occurs due to some rare thread +>> state, but I could not find any information that gives me any hint as to +>> what this state might be. +>> +>> So what I am asking is: +>> 1) Does anyone know why this sort of exception occurs? +>> 2) Can anyone think of a reason for such an exception to occur in the +>> situation I have described? +>> +>> Thanks in advance, +>> Nadav. +>> +>> +>> _______________________________________________ +>> sqlite-dev mailing list +>> sqlite-dev@sqlite.org +>> http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev +>> +>> +> +> _______________________________________________ +> sqlite-dev mailing list +> sqlite-dev@sqlite.org +> http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev +> +> + +------=_Part_29556_25702991.1217835502493 +Content-Type: text/html; charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +<div dir="ltr">Hi Grant,<br><br>Thanks for your reply.<br>I am using a different session for each thread, whenever a thread wishes to access the database it gets a session from the session pool and works with that session until its work is done.<br> +<br>Most of the actions the threads are doing on the database are quite complicated and are required to be fully committed or completely ignored, so yes, I am (most of the time) explicitly beginning and committing my transactions.<br> +<br>Regarding the SQLiteStatementImpl, I believe the Poco manual explains that sessions and statements for that matter cannot be shared between threads, therefore if you are using a session via one thread only it should work fine.<br> +<br>My first impression was that the problem was in the Poco infrastructure (I have found several Poco related bugs in the past), but the problem ALWAYS occurs when I perform the "BEGIN IMMEDIATE" action, if it were a Poco related bug, I would expect to see it here and there without any relation to this specific statement, but that is not the case.<br> +<br>None the less, I will also post my question on the Poco forums.<br><br>Nadav.<br><br><div class="gmail_quote">On Sun, Aug 3, 2008 at 5:47 PM, Grant Gatchel <span dir="ltr"><<a href="mailto:grant.gatchel@gmail.com">grant.gatchel@gmail.com</a>></span> wrote:<br> +<blockquote class="gmail_quote" style="border-left: 1px solid rgb(204, 204, 204); margin: 0pt 0pt 0pt 0.8ex; padding-left: 1ex;"><div dir="ltr">Are you using the same Poco::Session for every thread or does each call create a new session/handle to the database?<br> +<br>Are you explicitly BEGINning and COMMITting your transactions?<br><br>In looking at the 1.3.2 branch of Poco::Data::SQLite, there appears to be a race condition in the SQLiteStatementImpl::next() method in which the member _nextResponse is being accessed before the SQLiteStatementImpl::hasNext() method has a chance to interpret that value and throw an exception.<br> + +<br>This question might be more suitable in the Poco forums or mailinglist.<br><br>- Grant<br> +<br><div class="gmail_quote"><div><div></div><div class="Wj3C7c"> +On Sun, Aug 3, 2008 at 8:33 AM, nadav g <span dir="ltr"><<a href="http://nadav.gr" target="_blank">nadav.gr</a>@<a href="http://gmail.com" target="_blank">gmail.com</a>></span> wrote:<br></div></div><blockquote class="gmail_quote" style="border-left: 1px solid rgb(204, 204, 204); margin: 0pt 0pt 0pt 0.8ex; padding-left: 1ex;"> +<div><div></div><div class="Wj3C7c"> + + +<div dir="ltr">Hi All,<br><br>I have been using SQLite with Poco (<a href="http://www.appinf.com" target="_blank">www.appinf.com</a>) as my infrastructure.<br>The program is running several threads that access this database very often and are synchronized by SQLite itself.<br> + + + + +Everything seems to work just fine most of time (usually days - weeks) but I do get an occasional exception:<br><br>Exception: SQL error or missing database: Iterator Error: trying to check if there is a next value<br><br> + + + + +The backtrace leads to this statement:<br><b>"BEGIN IMMEDIATE"</b><br><br>This specific code runs numerous times before an exception occurs (if occurs at all) and I cannot think of any reason for it to fail later rather than sooner.<br> + + + + +It is pretty obvious that this situation occurs due to some rare thread state, but I could not find any information that gives me any hint as to what this state might be.<br><br>So what I am asking is:<br>1) Does anyone know why this sort of exception occurs?<br> + + + + +2) Can anyone think of a reason for such an exception to occur in the situation I have described?<br><br>Thanks in advance,<br>Nadav.<br><br></div> +<br></div></div>_______________________________________________<br> +sqlite-dev mailing list<br> +<a href="mailto:sqlite-dev@sqlite.org" target="_blank">sqlite-dev@sqlite.org</a><br> +<a href="http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev" target="_blank">http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev</a><br> +<br></blockquote></div><br></div> +<br>_______________________________________________<br> +sqlite-dev mailing list<br> +<a href="mailto:sqlite-dev@sqlite.org">sqlite-dev@sqlite.org</a><br> +<a href="http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev" target="_blank">http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev</a><br> +<br></blockquote></div><br></div> + +------=_Part_29556_25702991.1217835502493-- + +--===============2123623832== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev + +--===============2123623832==-- + diff --git a/testdata/testdir/cur/1220863087.12663_15.mindcrime!2,PS b/testdata/testdir/cur/1220863087.12663_15.mindcrime!2,PS new file mode 100644 index 0000000..d6487c0 --- /dev/null +++ b/testdata/testdir/cur/1220863087.12663_15.mindcrime!2,PS @@ -0,0 +1,136 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-3.6 required=3.0 tests=BAYES_00,RCVD_IN_DNSWL_LOW, + SPF_PASS,WHOIS_NETSOLPR autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 1A6CD69CB6 + for <xxxx@localhost>; Tue, 12 Aug 2008 21:42:38 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Tue, 12 Aug 2008 21:42:38 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs123119wfh; Sun, 10 Aug 2008 + 22:06:31 -0700 (PDT) +Received: by 10.100.166.10 with SMTP id o10mr9327844ane.0.1218431190107; Sun, + 10 Aug 2008 22:06:30 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id c29si10110392anc.13.2008.08.10.22.06.29; Sun, 10 Aug 2008 + 22:06:30 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:45637 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KSPbx-0006dj-96 for + xxxx.klub@gmail.com; Mon, 11 Aug 2008 01:06:29 -0400 +Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) id + 1KSPbE-0006cQ-Nd for help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 01:05:44 -0400 +Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) id + 1KSPbD-0006bs-Px for help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 01:05:44 -0400 +Received: from [199.232.76.173] (port=37426 helo=monty-python.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KSPbD-0006bk-HT for + help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 01:05:43 -0400 +Received: from main.gmane.org ([80.91.229.2]:46446 helo=ciao.gmane.org) by + monty-python.gnu.org with esmtps (TLS-1.0:RSA_AES_256_CBC_SHA1:32) (Exim + 4.60) (envelope-from <geh-help-gnu-emacs@m.gmane.org>) id 1KSPbD-0003Kl-CA + for help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 01:05:43 -0400 +Received: from list by ciao.gmane.org with local (Exim 4.43) id + 1KSPb9-00080r-CX for help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 05:05:39 +0000 +Received: from bas2-toronto63-1088792724.dsl.bell.ca ([64.229.168.148]) by + main.gmane.org with esmtp (Gmexim 0.1 (Debian)) id 1AlnuQ-0007hv-00 for + <help-gnu-emacs@gnu.org>; Mon, 11 Aug 2008 05:05:39 +0000 +Received: from cpchan by bas2-toronto63-1088792724.dsl.bell.ca with local + (Gmexim 0.1 (Debian)) id 1AlnuQ-0007hv-00 for <help-gnu-emacs@gnu.org>; Mon, + 11 Aug 2008 05:05:39 +0000 +X-Injected-Via-Gmane: http://gmane.org/ +To: help-gnu-emacs@gnu.org +From: anon@example.com +Date: Mon, 11 Aug 2008 01:03:22 -0400 +Organization: Linux Private Site +Message-ID: <87bq00nnxh.fsf@MagnumOpus.Mercurius> +References: <877iav5s49.fsf@163.com> <86hc9yc5sj.fsf@timbral.net> + <877iat7udd.fsf@163.com> <87fxphcsxi.fsf@lion.rapttech.com.au> + <8504ddd4-5e3b-4ed5-bf77-aa9cce81b59a@1g2000pre.googlegroups.com> + <87k5es59we.fsf@lion.rapttech.com.au> + <63c824e3-62b1-4a93-8fa8-2813e1f9397f@v13g2000pro.googlegroups.com> + <874p5vsgg8.fsf@nonospaz.fatphil.org> + <8250972e-1886-4021-80bc-376e34881c80@v39g2000pro.googlegroups.com> + <87zlnnqvvs.fsf@nonospaz.fatphil.org> + <57add0e0-b39d-4c71-8d2c-d3b9ddfaa1a9@1g2000pre.googlegroups.com> + <87sktfnz5p.fsf@atthis.clsnet.nl> + <562e1111-d9e7-4b6a-b661-3f9af13fea17@b30g2000prf.googlegroups.com> + <87d4khoq97.fsf@atthis.clsnet.nl> + <0fe404c5-cab8-4692-8a27-532e737a7813@i24g2000prf.googlegroups.com> +Mime-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; micalg=pgp-sha1; + protocol="application/pgp-signature" +X-Complaints-To: usenet@ger.gmane.org +X-Gmane-NNTP-Posting-Host: bas2-toronto63-1088792724.dsl.bell.ca +X-Face: G; + Z,`sm>)4t4LB/GUrgH$W`!AmfHMj,LG)Z}X0ax@s9:0>0)B&@vcm{v-le)wng)?|o]D<V6&ay<F=H{M5?$T%p!dPdJeF,au\E@TA"v22K!Zl\\mzpU4]6$ZnAI3_L)h; + fpd}mn2py/7gv^|*85-D_f:07cT>\Z}0:6X +User-Agent: Gnus/5.110011 (No Gnus v0.11) Emacs/23.0.60 (gnu/linux) +Cancel-Lock: sha1:IKyfrl5drOw6HllHFSmWHAKEeC8= +X-detected-kernel: by monty-python.gnu.org: Linux 2.6, seldom 2.4 (older, 4) +Subject: Re: Can anybody tell me how to send HTML-format mail in gnus +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 1229 +Lines: 36 + +--=-=-= +Content-Type: text/plain + +Xah <xahlee@gmail.com> writes: + +> So, i was reading about it in Wikipedia. Although i don't have a TV, +> and haven't had since 2000, but i still enjoyed the festive spirits +> anyhow. After all, i'm Chinese by blood. So, in my wandering, i ran +> into this welcome song on youtube: +> +> http://www.youtube.com/watch?v=1HEndNYVhZo + +What is your point? Your email is in plain text and I can click on the +link just fine- it is not exactly rocket science to implement parsing of +URL's to workable links in an Email program (a lot of programs does +that, including Gnus). Images can be included inline if you want. Also +mail markups such as *this*, **this** and _this_ have been around since +the Usenet days and displayed appropriately by a number of mailers. Like +others have said, most html messages that I have seen either contains +useless information, or are plain spam and can introduce a host of +security problems in some mailers. + +Charles + + +--=-=-= +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v2.0.4-svn0 (GNU/Linux) + +iD8DBQFIn8gm3epPyyKbwPYRApbvAKDRirXwzMzI+NHV77+QcP3EgTPaCgCfb/6m +GtNVKdYAeftaYm1nwRVoCDA= +=ULo3 +-----END PGP SIGNATURE----- +--=-=-=-- + + + diff --git a/testdata/testdir/cur/1220863087.12663_19.mindcrime!2,S b/testdata/testdir/cur/1220863087.12663_19.mindcrime!2,S new file mode 100644 index 0000000..78efa2a --- /dev/null +++ b/testdata/testdir/cur/1220863087.12663_19.mindcrime!2,S @@ -0,0 +1,77 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id C4D6569CB3 + for <xxxx@localhost>; Thu, 7 Aug 2008 08:10:08 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Thu, 07 Aug 2008 08:10:08 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs34794wfh; Wed, 6 Aug 2008 + 13:40:29 -0700 (PDT) +Received: by 10.100.33.13 with SMTP id g13mr1093301ang.79.1218055228418; Wed, + 06 Aug 2008 13:40:28 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d19si15908789and.17.2008.08.06.13.40.27; Wed, 06 Aug 2008 + 13:40:28 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:56316 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQpo3-0007Pc-Qk for + xxxx.klub@gmail.com; Wed, 06 Aug 2008 16:40:27 -0400 +From: anon@example.com +Newsgroups: gnu.emacs.help +Date: Wed, 6 Aug 2008 20:38:35 +0100 +Message-ID: <r6bpm5-6n6.ln1@news.ducksburg.com> +References: <55dbm5-qcl.ln1@news.ducksburg.com> + <mailman.15710.1217599959.18990.help-gnu-emacs@gnu.org> +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +X-Trace: individual.net bABVU1hcJwWAuRwe/097AAoOXnGGeYR8G1In635iFGIyfDLPUv +X-Orig-Path: news.ducksburg.com!news +Cancel-Lock: sha1:wK7dsPRpNiVxpL/SfvmNzlvUR94= + sha1:oepBoM0tJBLN52DotWmBBvW5wbg= +User-Agent: slrn/pre0.9.9-120/mm/ao (Ubuntu Hardy) +Path: news.stanford.edu!headwall.stanford.edu!newshub.sdsu.edu!feeder.erje.net!proxad.net!feeder1-2.proxad.net!feed.ac-versailles.fr!fu-berlin.de!uni-berlin.de!individual.net!not-for-mail +Xref: news.stanford.edu gnu.emacs.help:160868 +To: help-gnu-emacs@gnu.org +Subject: Re: Learning LISP; Scheme vs elisp. +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 417 +Lines: 11 + +On 2008-08-01, Thien-Thi Nguyen wrote: + +> warriors attack, felling foe after foe, +> few growing old til they realize: to know +> what deceit is worth deflection; +> such receipt reversed rejection! +> then their heavy arms, e'er transformed to shields: +> balanced hooked charms, ploughed deep, rich yields. + +Aha: the exercise for the reader is to place the parens correctly. +Might take me a while to solve this puzzle. + diff --git a/testdata/testdir/cur/1220863087.12663_5.mindcrime!2,S b/testdata/testdir/cur/1220863087.12663_5.mindcrime!2,S new file mode 100644 index 0000000..de46cc8 --- /dev/null +++ b/testdata/testdir/cur/1220863087.12663_5.mindcrime!2,S @@ -0,0 +1,84 @@ +Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 32F276963F + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:34 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Mon, 04 Aug 2008 21:49:34 +0300 (EEST) +Received: by 10.142.51.12 with SMTP id y12cs89397wfy; Mon, 4 Aug 2008 02:41:16 + -0700 (PDT) +Received: by 10.150.156.20 with SMTP id d20mr963580ybe.104.1217842875596; Mon, + 04 Aug 2008 02:41:15 -0700 (PDT) +Received: from sqlite.org (sqlite.org [67.18.92.124]) by mx.google.com with + ESMTP id 6si3605185ywi.1.2008.08.04.02.40.57; Mon, 04 Aug 2008 02:41:15 -0700 + (PDT) +Received-SPF: pass (google.com: best guess record for domain of + sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as permitted sender) + client-ip=67.18.92.124; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record + for domain of sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as + permitted sender) smtp.mail=sqlite-dev-bounces@sqlite.org +Received: from sqlite.org (localhost [127.0.0.1]) by sqlite.org (Postfix) with + ESMTP id 7147F11C45; Mon, 4 Aug 2008 05:40:55 -0400 (EDT) +X-Original-To: sqlite-dev@sqlite.org +Delivered-To: sqlite-dev@sqlite.org +Received: from relay00.pair.com (relay00.pair.com [209.68.5.9]) by sqlite.org + (Postfix) with SMTP id B5F901192C for <sqlite-dev@sqlite.org>; Mon, 4 Aug + 2008 05:40:52 -0400 (EDT) +Received: (qmail 59961 invoked from network); 4 Aug 2008 09:40:50 -0000 +Received: from unknown (HELO ?192.168.0.17?) (unknown) by unknown with SMTP; 4 + Aug 2008 09:40:50 -0000 +X-pair-Authenticated: 87.13.75.164 +Message-Id: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: anon@example.com +To: sqlite-dev@sqlite.org +Mime-Version: 1.0 (Apple Message framework v926) +Date: Mon, 4 Aug 2008 11:40:49 +0200 +X-Mailer: Apple Mail (2.926) +Subject: [sqlite-dev] VM optimization inside sqlite3VdbeExec +X-BeenThere: sqlite-dev@sqlite.org +X-Mailman-Version: 2.1.9 +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +List-Unsubscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=unsubscribe> +List-Archive: <http://sqlite.org:8080/cgi-bin/mailman/private/sqlite-dev> +List-Post: <mailto:sqlite-dev@sqlite.org> +List-Help: <mailto:sqlite-dev-request@sqlite.org?subject=help> +List-Subscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=subscribe> +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Sender: sqlite-dev-bounces@sqlite.org +Errors-To: sqlite-dev-bounces@sqlite.org +Content-Length: 639 + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; +--- +Marco Bambini +http://www.sqlabs.net +http://www.sqlabs.net/blog/ +http://www.sqlabs.net/realsqlserver/ + + + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev + diff --git a/testdata/testdir/cur/1220863087.12663_7.mindcrime!2,RS b/testdata/testdir/cur/1220863087.12663_7.mindcrime!2,RS new file mode 100644 index 0000000..b5c0651 --- /dev/null +++ b/testdata/testdir/cur/1220863087.12663_7.mindcrime!2,RS @@ -0,0 +1,138 @@ +Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 3EBAB6963B + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:35 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Mon, 04 Aug 2008 21:49:35 +0300 (EEST) +Received: by 10.142.51.12 with SMTP id y12cs89536wfy; Mon, 4 Aug 2008 02:48:56 + -0700 (PDT) +Received: by 10.150.134.21 with SMTP id h21mr7950048ybd.181.1217843335665; + Mon, 04 Aug 2008 02:48:55 -0700 (PDT) +Received: from sqlite.org (sqlite.org [67.18.92.124]) by mx.google.com with + ESMTP id 6si5897081ywi.1.2008.08.04.02.48.35; Mon, 04 Aug 2008 02:48:55 -0700 + (PDT) +Received-SPF: pass (google.com: best guess record for domain of + sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as permitted sender) + client-ip=67.18.92.124; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record + for domain of sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as + permitted sender) smtp.mail=sqlite-dev-bounces@sqlite.org +Received: from sqlite.org (localhost [127.0.0.1]) by sqlite.org (Postfix) with + ESMTP id ED01611C4E; Mon, 4 Aug 2008 05:48:31 -0400 (EDT) +X-Original-To: sqlite-dev@sqlite.org +Delivered-To: sqlite-dev@sqlite.org +Received: from mx0.security.ro (mx0.security.ro [80.96.72.194]) by sqlite.org + (Postfix) with ESMTP id EB3F51192C for <sqlite-dev@sqlite.org>; Mon, 4 Aug + 2008 05:48:28 -0400 (EDT) +Received: (qmail 348 invoked from network); 4 Aug 2008 12:48:03 +0300 +Received: from dev.security.ro (HELO ?192.168.1.70?) (192.168.1.70) by + mx0.security.ro with SMTP; 4 Aug 2008 12:48:03 +0300 +Message-ID: <4896D06A.8000901@security.ro> +Date: Mon, 04 Aug 2008 12:48:26 +0300 +From: anon@example.com +User-Agent: Thunderbird 2.0.0.16 (Windows/20080708) +MIME-Version: 1.0 +To: sqlite-dev@sqlite.org +References: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +In-Reply-To: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +Content-Type: multipart/mixed; boundary="------------000207070200050102060301" +X-BitDefender-Scanner: Clean, Agent: BitDefender qmail 2.0.0 on + mx0.security.ro +X-BitDefender-Spam: No (0) +X-BitDefender-SpamStamp: v1, whitelisted, total: 0 +Subject: Re: [sqlite-dev] VM optimization inside sqlite3VdbeExec +X-BeenThere: sqlite-dev@sqlite.org +X-Mailman-Version: 2.1.9 +Precedence: high +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +List-Unsubscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=unsubscribe> +List-Archive: <http://sqlite.org:8080/cgi-bin/mailman/private/sqlite-dev> +List-Post: <mailto:sqlite-dev@sqlite.org> +List-Help: <mailto:sqlite-dev-request@sqlite.org?subject=help> +List-Subscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=subscribe> +Sender: sqlite-dev-bounces@sqlite.org +Errors-To: sqlite-dev-bounces@sqlite.org +Content-Length: 2212 + +This is a multi-part message in MIME format. +--------------000207070200050102060301 +Content-Type: text/plain; charset=ISO-8859-1; format=flowed +Content-Transfer-Encoding: 7bit + +Marco Bambini wrote: +> Inside sqlite3VdbeExec there is a very big switch statement. +> In order to increase performance with few modifications to the +> original code, why not use this technique ? +> http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html +> +> With a properly defined "instructions" array, instead of the switch +> statement you can use something like: +> goto * instructions[pOp->opcode]; +> --- +> Marco Bambini +> http://www.sqlabs.net +> http://www.sqlabs.net/blog/ +> http://www.sqlabs.net/realsqlserver/ +> +> +> +> _______________________________________________ +> sqlite-dev mailing list +> sqlite-dev@sqlite.org +> http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev +> +All the world's not a VAX. This technique is GCC-specific. The SQLite +source must be as portable as possible thus tying it to a specific +compiler is out of the question. While one could conceivably use some +preprocessor magic to provide alternate implementations, that would be +impractical considering the sheer size of the code affected. +On the other hand - perhaps you could benchmark the change and provide +some data on whether this actually improves performance? + + +--------------000207070200050102060301 +Content-Type: text/x-vcard; charset=utf-8; + name="mihailim.vcf" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; + filename="mihailim.vcf" + +begin:vcard +fn:Mihai Limbasan +n:Limbasan;Mihai +org:SC SECPRAL COM SRL +adr:;;str. Actorului nr. 9;Cluj-Napoca;Cluj;400441;Romania +email;internet:mihailim@security.ro +title:SoftwareDeveloper +tel;work:+40 264 449579 +tel;fax:+40 264 418594 +tel;cell:+40 729 038302 +url:http://secpral.ro/ +version:2.1 +end:vcard + + +--------------000207070200050102060301 +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev + +--------------000207070200050102060301-- + diff --git a/testdata/testdir/cur/1252168370_3.14675.cthulhu!2,S b/testdata/testdir/cur/1252168370_3.14675.cthulhu!2,S new file mode 100644 index 0000000..4fad706 --- /dev/null +++ b/testdata/testdir/cur/1252168370_3.14675.cthulhu!2,S @@ -0,0 +1,21 @@ +Return-Path: <dfgh@floppydisk.nl> +X-Spam-Checker-Version: SpamAssassin 3.1.0 (2005-09-13) on mindcrime +X-Spam-Level: +Delivered-To: dfgh@floppydisk.nl +Message-ID: <43A09C49.9040902@euler.org> +Date: Wed, 14 Dec 2005 23:27:21 +0100 +From: Fred Flintstone <fred@euler.org> +User-Agent: Mozilla Thunderbird 1.0.7 (X11/20051010) +X-Accept-Language: nl-NL, nl, en +MIME-Version: 1.0 +To: dfgh@floppydisk.nl +Subject: Re: xyz +References: <439C1136.90504@euler.org> <4399DD94.5070309@euler.org> <20051209233303.GA13812@gauss.org> <439B41ED.2080402@euler.org> <4399DD94.5070309@euler.org> <20051209233303.GA13812@gauss.org> <439A1E03.3090604@euler.org> <20051211184308.GB13513@gauss.org> +In-Reply-To: <20051211184308.GB13513@gauss.org> +X-Enigmail-Version: 0.92.0.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit +X-UIDL: T<?"!%LG"!cAK"!_j(#! +Content-Length: 1879 + +Test 123. diff --git a/testdata/testdir/cur/1283599333.1840_11.cthulhu!2, b/testdata/testdir/cur/1283599333.1840_11.cthulhu!2, new file mode 100644 index 0000000..25c7180 --- /dev/null +++ b/testdata/testdir/cur/1283599333.1840_11.cthulhu!2, @@ -0,0 +1,16 @@ +From: Frodo Baggins <frodo@example.com> +To: Bilbo Baggins <bilbo@anotherexample.com> +Subject: Greetings from =?UTF-8?B?TG90aGzDs3JpZW4=?= +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +Fcc: .sent +Organization: The Fellowship of the Ring +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Message-Id: <abcd$efgh@example.com> + + +Let's write some fünkÿ text +using umlauts. + +Foo. diff --git a/testdata/testdir/cur/1305664394.2171_402.cthulhu!2, b/testdata/testdir/cur/1305664394.2171_402.cthulhu!2, new file mode 100644 index 0000000..863f714 --- /dev/null +++ b/testdata/testdir/cur/1305664394.2171_402.cthulhu!2, @@ -0,0 +1,17 @@ +From: =?UTF-8?B?TcO8?= <testmu@testmu.xx> +To: Helmut =?UTF-8?B?S3LDtmdlcg==?= <hk@testmu.xxx> +Subject: =?UTF-8?B?TW90w7ZyaGVhZA==?= +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +References: <non-exist-01@msg.id> <non-exist-02@msg.id> <non-exist-03@msg.id> <non-exist-04@msg.id> +1n-Reply-To: <non-exist-04@msg.id> +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + + +Test for issue #38, where apparently searching for accented words in subject, +to etc. fails. + +What about here? Queensrÿche. Mötley Crüe. + + diff --git a/testdata/testdir/cur/encrypted!2,S b/testdata/testdir/cur/encrypted!2,S new file mode 100644 index 0000000..f75fd40 --- /dev/null +++ b/testdata/testdir/cur/encrypted!2,S @@ -0,0 +1,56 @@ +Return-path: <> +Envelope-to: peter@example.com +Delivery-date: Fri, 11 May 2012 16:22:03 +0300 +Received: from localhost.example.com ([127.0.0.1] helo=borealis) + by borealis with esmtp (Exim 4.77) + id 1SSpnB-00038a-Ux + for djcb@localhost; Fri, 11 May 2012 16:21:58 +0300 +Delivered-To: peter@example.com +From: Brian <brian@example.com> +To: Peter <peter@example.com> +Subject: encrypted +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:21:42 +0300 +Message-ID: <!&!AAAAAAAAAYAAAAAAAAAOH1+8mkk+lLn7Gg5fke7FbCgAAAEAAAAJ7eBDgcactKhXL6r8cEnJ8BAAAAAA==@example.com> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +hQQOA1T38TPQrHD6EA//YXkUB4Dy09ngCRyHWbXmV3XBjuKTr8xrak5ML1kwurav +gyagOHKLMU+5CKvObChiKtXhtgU0od7IC8o+ALlHevQ0XXcqNYA2KUfX8R7akq7d +Xx9mA6D8P7Y/P8juUCLBpfrCi2GC42DtvPZSUu3bL/ctUJ3InPHIfHibKF2HMm7/ +gUHAKY8VPJF39dLP8GLcfki6qFdeWbxgtzmuyzHfCBCLnDL0J9vpEQBpGDFMcc4v +cCbmMJaiPOmRb6U4WOuRVnuXuTztLiIn0jMslzOSFDcLTVBAsrC01r71O+XZKfN4 +mIfcpcWJYKM2NQW8Jwf+8Hr84uznBqs8uTTlrmppjkAHZGqGMjiQDxLhDVaCQzMy +O8PSV4xT6HPlKXOwV1OLc+vm0A0RAdSBctgZg40oFn4XdB1ur8edwAkLvc0hJKaz +gyTQiPaXm2Uh2cDeEx4xNgXmwCKasqc9jAlnDC2QwA33+pw3OqgZT5h1obn0fAeR +mgB+iW1503DIi/96p8HLZcr2EswLEH9ViHIEaFj/vlR5BaOncsLB0SsNV/MHRvym +Xg5GUjzPIiyBZ3KaR9OIBiZ5eXw+bSrPAo/CAs0Zwxag7W3CH//oK39Qo1GnkYpc +4IQxhx4IwkzqtCnripltV/kfpGu0yA/OdK8lOjkUqCwvL97o73utXIxm21Zd3mEP +/iLNrduZjMCq+goz1pDAQa9Dez6VjwRuRPTqeAac8Fx/nzrVzIoIEAt36hpuaH1l +KpbmHpKgsUWcrE5iYT0RRlRRtRF4PfJg8PUmP1hvw8TaEmNfT+0HgzcJB/gRsVdy +gTzkzUDzGZLhRcpmM5eW4BkuUmIO7625pM6Jd3HOGyfCGSXyEZGYYeVKzv8xbzYf +QM6YYKooRN9Ya2jdcWguW0sCSJO/RZ9eaORpTeOba2+Fp6w5L7lga+XM9GLfgref +Cf39XX1RsmRBsrJTw0z5COf4bT8G3/IfQP0QyKWIFITiFjGmpZhLsKQ3KT4vSe/d +gTY1xViVhkjvMFn3cgSOSrvktQpAhsXx0IRazN0T7pTU33a5K0SrZajY9ynFDIw9 +we7XYyVwZzYEXjGih5mTH1PhWYK5fZZEKKqaz5TyYv9SeWJ+8FrHeXUKD38SQEHM +qkpl9Iv17RF4Qy9uASWwRoobhKO+GykTaBSTyw8R8ctG/hfAlnaZxQ3TwNyHWyvU +9SVJsp27ulv/W9MLZtGpEMK0ckAR164Vyou1KOn200BqxbC2tJpegNeD2TP5ZtdY +HIcxkgKr0haYcDnVEf1ulSxv23pZWIexbgvVCG7dRL0eB+6O28f9CWehle10MDyM +0AYyw8Da2cu7PONMovqt4nayScyGTacFBp7c2KXR9DGZ0mcBwOjL/mGRKcVWN3MG +2auCrwn2KVWmKZI3Jp0T8KhfGBnFs9lUElpDTOiED1/2bKz6Yoc385QtWx99DFMZ +IWiH5wMxkWFpzjE+GHiJ09vSbTTL4JY9eu2n5nxQmtjYMBVxQm7S7qwH +=0Paa +-----END PGP MESSAGE----- +--=-=-=-- diff --git a/testdata/testdir/cur/multimime!2,FS b/testdata/testdir/cur/multimime!2,FS new file mode 100644 index 0000000..84f85aa --- /dev/null +++ b/testdata/testdir/cur/multimime!2,FS @@ -0,0 +1,27 @@ +Return-path: <> +Envelope-to: djcb@localhost +Delivery-date: Sun, 20 May 2012 09:59:51 +0300 +From: Steve Jobs <jobs@example.com> +To: Bill Gates <bg@example.com> +Subject: multimime +User-agent: mu4e 0.9.8.4; emacs 23.3.1 +Date: Sat, 19 May 2012 20:57:56 +0100 +Message-ID: <m2fwaw2baz.fsf@example.com> +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain + +abc +--=-=-= +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="test1.C" +Content-Transfer-Encoding: base64 + +aGVyZSBpcyBhIHNpbXBsZSB0ZXN0IGZpbGUuCg== +--=-=-= +Content-Type: text/plain + +def +--=-=-=-- diff --git a/testdata/testdir/cur/multirecip!2,S b/testdata/testdir/cur/multirecip!2,S new file mode 100644 index 0000000..c997503 --- /dev/null +++ b/testdata/testdir/cur/multirecip!2,S @@ -0,0 +1,11 @@ +Date: Thu, 15 May 2016 14:57:25 -0200 +From: +To: a@example.com,b@example.com,c@example.com +Cc: d@example.com,e@example.com +Subject: test with multi to and cc +Message-id: <3BE9E652343245@emss35m06.us.lmco.com> + +Message with multi cc and to. + + + diff --git a/testdata/testdir/cur/signed!2,S b/testdata/testdir/cur/signed!2,S new file mode 100644 index 0000000..a2e7e21 --- /dev/null +++ b/testdata/testdir/cur/signed!2,S @@ -0,0 +1,36 @@ +Return-path: <> +Envelope-to: skipio@localhost +Delivery-date: Fri, 11 May 2012 16:21:57 +0300 +Received: from localhost.roma.net([127.0.0.1] helo=borealis) + by borealis with esmtp (Exim 4.77) + id 1SSpnB-00038a-55 + for djcb@localhost; Fri, 11 May 2012 16:21:57 +0300 +Delivered-To: diggler@gmail.com +From: Skipio <skipio@roma.net> +To: Hannibal <hanni@carthago.net> +Subject: signed +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:20:45 +0300 +Message-ID: <878vgy97ma.fsf@roma.net> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; micalg=pgp-sha1; + protocol="application/pgp-signature" + +--=-=-= +Content-Type: text/plain + + +I am signed! + +--=-=-= +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +iEYEARECAAYFAk+tEi0ACgkQ6WrHoQF92jxTzACeKd/XxY+P7bpymWL3JBRHaW9p +DpwAoKw7PDW4z/lNTkWjndVTjoO9jGhs +=blXz +-----END PGP SIGNATURE----- +--=-=-=-- + diff --git a/testdata/testdir/cur/signed-encrypted!2,S b/testdata/testdir/cur/signed-encrypted!2,S new file mode 100644 index 0000000..a3910e6 --- /dev/null +++ b/testdata/testdir/cur/signed-encrypted!2,S @@ -0,0 +1,54 @@ +Return-path: <> +Envelope-to: karjala@localhost +Delivery-date: Fri, 11 May 2012 16:37:57 +0300 +From: karjala@example.com +To: lapinkulta@example.com +Subject: signed + encrypted +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:36:08 +0300 +Message-ID: <874nrm96wn.fsf@example.com> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +hQQOA1T38TPQrHD6EA/+K4kSpMa7zk+qihUkQnHSq28xYxisNQx6X5DVNjA/Qx16 +uZj/40ae+PoSMTVfklP+B2S/IomuTW6dwVqS7aQ3u4MTzi+YOi11k1lEMD7hR0Wb +L0i48o3/iCPuCTpnOsaLZvRL06g+oTi0BF2pgz/YdsgsBTGrTb3pkDGSlLIhvh/J +P8eE3OuzkXS6d8ymJKx2S2wQJrc1AFf1BgJfgc5T0iAvcV+zIMG+PIYcVd04zVpj +cORFEfvGgfxWkeX+Ks3tu/l5PA1EesnoqFdNFZm+RKBg3RFsOm8tBlJ46xJjfeHg +zLgifeSLy3tOX7CvWYs9torrx7s7UOI2gV8kzBqz+a7diyCMezceeQ9l0nIRybwW +C9Egp8Bpfb02iXTOGdE/vRiNItQH14GKmXf4nCSwdtQUm3yzaqY9yL3xBxAlW53e +YOFfPMESt+E7IlPn0c7llWGrcdrhJbUEoGOIPezES7kdeNPzi8G1lLtvT04/SSZJ +QxPH5FNzSFaYFAQSdI7TR69P7L7vtLL8ndkjY49HfLFXochQQzsqrzVxzRCruHxA +zbZSRptNf9SuXEaX9buO1vlFHheGvrCKzEWa6O7JD/DiyrE/zqy4jdlh9abMCouQ +GWGSbn8jk6SMTQQ2Yv/VOyFqifHZp0UJD59tyIdenpxoYu5M0lwHLNVDlRjLEwUQ +AIDz1tbLoM7lxs2FOKGr8QqbKIeMfL+NUmbvVIDc4mJrOlRnHh+cZYm4Z49iTl1v +bYNMYgR5nY7W6rqh0ae7ZOW0h2NzpkAwTzuf1YrSjNavd9KBwOCFtAoZhRwfwFVx +ju+ByHFNnf7g/R6DekHS0pSiatM0cPDJT05atEZb+13CRHHznonmLHi+VahXjrpg +cIUA8Lhjdfm6Fsabo7gNZnTTRxNBqUXKK2vJF/XLbNrH5K2BH2dCCmUNtm3yFWiM +DOzaw3665Y3S6MvZdyKpatbNrVoJdBpRgPxJ1YCSEituFUqHJBStay+aRb5fVkQR +w3+9hWw+Ob0+2EumKbgfQ7iMwTZBCZP4VOxkoqdHvs9aWm4N7wHtXsyCew3icbJx +lyUWsDx/FI+HlQRfOqeAMxmp8kKybmHNw8oGiw+uPPUHSD1NFYVm2DtwhYll3Fvs +YY7r5s3yP1ZnwxMqWI3OsExVUXs8MS4UTAgO+cggO7YidPcANbBDihBFP8mTXtni +Oo5n5v+/eRoLfHmnsGcaK8EkKsfFHpbqn4gxXGcBuHaTTJ/ZhbW6bi1WWZA9ExaJ +IeTDtp5Bks1pJvTjCDacvgwl3rEBM6yaeIvB7575Y/GPMTOZhawhfOxV1smMmTKI +JOWYb3+PuN2cvWetkjFgH8re4sRXq22DKBZHJEWYU8sH0sACAePnIr+pkrOtGeJB +t1zBqZUnrupH6ptk9n/AjbQ+XSMTEKu55gSjYLAYx1EHApx52QLkdh+ej5xCIVeY +6wS1Iipkoc6/r6F7CKctupXurNY2AlD4uQIOfD6kQgkqK4PY3hsRHQA+Zqj6oRfr +kxysFJZvhgt26IeBVapFs10WuYt9iHfpbPUBQUIZCLyPAh08UdVW64Uc2DvUPy+I +C+3RrmTHQPP/YNKgDQaZ3ySVEDkqjaDPmXr5K0Ibaib2dtPCLcA= +=pv03 +-----END PGP MESSAGE----- +--=-=-=-- + diff --git a/testdata/testdir/cur/special!2,Sabc b/testdata/testdir/cur/special!2,Sabc new file mode 100644 index 0000000..7f1de8e --- /dev/null +++ b/testdata/testdir/cur/special!2,Sabc @@ -0,0 +1,10 @@ +Date: Thu, 1 Jun 2012 14:57:25 -0200 +From: "Rocky Balboa" <rocky@example.com> +To: "Ivan Drago" <ivan@example.com> +Subject: currying and tail optimization +Message-id: <3BE9E653ef345@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT + +Test 123. I'm a special message with special flags. diff --git a/testdata/testdir/new/1220863087.12663_21.mindcrime b/testdata/testdir/new/1220863087.12663_21.mindcrime new file mode 100644 index 0000000..4101716 --- /dev/null +++ b/testdata/testdir/new/1220863087.12663_21.mindcrime @@ -0,0 +1,111 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 6389969CB2 + for <xxxx@localhost>; Thu, 7 Aug 2008 08:10:07 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Thu, 07 Aug 2008 08:10:07 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs34769wfh; Wed, 6 Aug 2008 + 13:38:53 -0700 (PDT) +Received: by 10.100.6.13 with SMTP id 13mr4103508anf.83.1218055131215; Wed, 06 + Aug 2008 13:38:51 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id b32si10199298ana.34.2008.08.06.13.38.49; Wed, 06 Aug 2008 + 13:38:51 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +DomainKey-Status: good (test mode) +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org; domainkeys=pass + (test mode) header.From=juanma_bellon@yahoo.es +Received: from localhost ([127.0.0.1]:55648 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQpmT-0005W9-AQ for + xxxx.klub@gmail.com; Wed, 06 Aug 2008 16:38:49 -0400 +Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) id + 1KQplz-0005U5-Pk for help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:38:19 -0400 +Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) id + 1KQplw-0005Nw-OG for help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:38:19 -0400 +Received: from [199.232.76.173] (port=45465 helo=monty-python.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQplw-0005NX-I6 for + help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:38:16 -0400 +Received: from n74a.bullet.mail.sp1.yahoo.com ([98.136.45.21]:29868) by + monty-python.gnu.org with smtp (Exim 4.60) (envelope-from + <juanma_bellon@yahoo.es>) id 1KQplw-0007EF-7Z for help-gnu-emacs@gnu.org; + Wed, 06 Aug 2008 16:38:16 -0400 +Received: from [216.252.122.216] by n74.bullet.mail.sp1.yahoo.com with NNFMP; + 06 Aug 2008 20:38:14 -0000 +Received: from [68.142.237.89] by t1.bullet.sp1.yahoo.com with NNFMP; 06 Aug + 2008 20:38:14 -0000 +Received: from [69.147.75.180] by t5.bullet.re3.yahoo.com with NNFMP; 06 Aug + 2008 20:38:14 -0000 +Received: from [127.0.0.1] by omp101.mail.re1.yahoo.com with NNFMP; 06 Aug + 2008 20:38:14 -0000 +X-Yahoo-Newman-Id: 778995.62909.bm@omp101.mail.re1.yahoo.com +Received: (qmail 43643 invoked from network); 6 Aug 2008 20:38:14 -0000 +DomainKey-Signature: a=rsa-sha1; q=dns; c=nofws; s=s1024; d=yahoo.es; + h=Received:X-YMail-OSG:X-Yahoo-Newman-Property:From:To:Subject:Date:User-Agent:References:In-Reply-To:MIME-Version:Content-Type:Content-Transfer-Encoding:Content-Disposition:Message-Id; + b=ThdHlND5CNUsLPGuk+XhCWkdUA9w7lg4hiAgx8F8egsmQteMpwUlV/Y5tfe6K3O2jzHjtsklkzWqm7WY3VAcxxD/QgxLnianK5ZQHoelDAiGaFRqu8Y42XMZso2ccCBFWUQaKo9C+KIfa3e3ci73qehVxTtmr7bxLjurcSYEBPo= + ; +Received: from unknown (HELO 212251170160.customer.cdi.no) + (juanma_bellon@212.251.170.160 with plain) by smtp109.plus.mail.re1.yahoo.com + with SMTP; 6 Aug 2008 20:38:14 -0000 +X-YMail-OSG: k86L54kVM1kiZbUlYx7gayoBrCLYMFIRDL.KJLBKetNucAbwU4RjeeE1vhjw33hREaUig0CCjG7BTwIfbeZZpRmUcHbxl6gR0z6Sd3lYqA-- +X-Yahoo-Newman-Property: ymail-3 +From: anon@example.com +To: help-gnu-emacs@gnu.org +Date: Wed, 6 Aug 2008 22:38:15 +0200 +User-Agent: KMail/1.9.6 (enterprise 0.20070907.709405) +References: <mailman.15123.1216681940.18990.help-gnu-emacs@gnu.org> + <mailman.15143.1216715014.18990.help-gnu-emacs@gnu.org> + <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> +In-Reply-To: <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: quoted-printable +Content-Disposition: inline +Message-Id: <200808062238.15634.juanma_bellon@yahoo.es> +X-detected-kernel: by monty-python.gnu.org: FreeBSD 6.x (1) +Subject: Re: basic question: going back to dired +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 361 + +On Thursday 31 July 2008, Xah wrote: +> what's the logic of =E2=80=9COK=E2=80=9D? + +=46or all I know, it comes from "0 Knock-outs" (from USA civil war times, +IIRC), i.e., all went really well. + +But this is really off-topic. +=2D-=20 +Juanma + +"Having a smoking section in a restaurant is like + having a peeing section in a swimming pool." + -- Edward Burr + + + + + diff --git a/testdata/testdir/new/1220863087.12663_23.mindcrime b/testdata/testdir/new/1220863087.12663_23.mindcrime new file mode 100644 index 0000000..ca46f2b --- /dev/null +++ b/testdata/testdir/new/1220863087.12663_23.mindcrime @@ -0,0 +1,105 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id C3EF069CB3 + for <xxxx@localhost>; Thu, 7 Aug 2008 08:10:10 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Thu, 07 Aug 2008 08:10:10 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs35153wfh; Wed, 6 Aug 2008 + 13:58:17 -0700 (PDT) +Received: by 10.100.166.10 with SMTP id o10mr4182182ane.0.1218056296101; Wed, + 06 Aug 2008 13:58:16 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d34si13875743and.3.2008.08.06.13.58.14; Wed, 06 Aug 2008 + 13:58:16 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org; dkim=pass (test + mode) header.i=@gmail.com +Received: from localhost ([127.0.0.1]:33418 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQq5G-0001aY-Cr for + xxxx.klub@gmail.com; Wed, 06 Aug 2008 16:58:14 -0400 +Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) id + 1KQq4n-0001Z9-06 for help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:57:45 -0400 +Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) id + 1KQq4l-0001V8-6c for help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:57:44 -0400 +Received: from [199.232.76.173] (port=46438 helo=monty-python.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQq4k-0001Un-V2 for + help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:57:42 -0400 +Received: from ik-out-1112.google.com ([66.249.90.180]:17562) by + monty-python.gnu.org with esmtp (Exim 4.60) (envelope-from + <lekktu@gmail.com>) id 1KQq4k-0001fk-OW for help-gnu-emacs@gnu.org; Wed, 06 + Aug 2008 16:57:42 -0400 +Received: by ik-out-1112.google.com with SMTP id c21so94956ika.2 for + <help-gnu-emacs@gnu.org>; Wed, 06 Aug 2008 13:57:41 -0700 (PDT) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=gamma; + h=domainkey-signature:received:received:message-id:date:from:to + :subject:cc:in-reply-to:mime-version:content-type + :content-transfer-encoding:content-disposition:references; + bh=TTNY9749hpg1+TXOwdaCr+zbQGhBUt3IvsjLWp+pxp0=; + b=BOfudUT/SiW9V4e9+k3dXDzwm+ogdrq4m5OlO+f1H+oE6OAYGIm8dbdqDAOwUewBoS + jRpfZo07YamP9rkko79SeFdQnf7UAPFAw9x7DFCm3x6muSlCcJBR7vYs1rgHOSINAn2B + vQx2//lKR4fXfKNURNu+B30KrvoEmw6m2C8dI= +DomainKey-Signature: a=rsa-sha1; c=nofws; d=gmail.com; s=gamma; + h=message-id:date:from:to:subject:cc:in-reply-to:mime-version + :content-type:content-transfer-encoding:content-disposition :references; + b=UMDBulH/LwxDywEH0pfK3DbJ4u2kIZCVDLIM++PqrdcR82HjcS/O3Jhf5OFrf7Fnyj + GH76xmc7zkTG/3aQy2WY6DeWCJaFarEItmhxy3h/xS+kUKeDARzNox0OzK6lIv/u9bdy + f2LnFlYRJ7Q5vy3lxpxAWB4v0qCwtF9LjWFg4= +Received: by 10.210.47.7 with SMTP id u7mr3100239ebu.30.1218056261587; Wed, 06 + Aug 2008 13:57:41 -0700 (PDT) +Received: by 10.210.71.14 with HTTP; Wed, 6 Aug 2008 13:57:41 -0700 (PDT) +Message-ID: <f7ccd24b0808061357t453f5962w8b61f9a453b684d0@mail.gmail.com> +Date: Wed, 6 Aug 2008 22:57:41 +0200 +From: anon@example.com +To: Juanma <juanma_bellon@yahoo.es> +In-Reply-To: <200808062238.15634.juanma_bellon@yahoo.es> +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline +References: <mailman.15123.1216681940.18990.help-gnu-emacs@gnu.org> + <mailman.15143.1216715014.18990.help-gnu-emacs@gnu.org> + <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> + <200808062238.15634.juanma_bellon@yahoo.es> +X-detected-kernel: by monty-python.gnu.org: Linux 2.6 (newer, 2) +Cc: help-gnu-emacs@gnu.org +Subject: Re: basic question: going back to dired +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 309 + +On Wed, Aug 6, 2008 at 22:38, Juanma <juanma_bellon@yahoo.es> wrote: + +> For all I know, it comes from "0 Knock-outs" (from USA civil war times, +> IIRC), i.e., all went really well. + +See http://en.wikipedia.org/wiki/Okay#Etymology + +"0 knock-outs" is among the "Improbable or refuted etymologies". + + Juanma + + diff --git a/testdata/testdir/new/1220863087.12663_25.mindcrime b/testdata/testdir/new/1220863087.12663_25.mindcrime new file mode 100644 index 0000000..588ace1 --- /dev/null +++ b/testdata/testdir/new/1220863087.12663_25.mindcrime @@ -0,0 +1,98 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-3.6 required=3.0 tests=BAYES_00,RCVD_IN_DNSWL_LOW, + SPF_PASS autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id D68E769CB5 + for <xxxx@localhost>; Fri, 8 Aug 2008 20:56:25 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Fri, 08 Aug 2008 20:56:25 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs71287wfh; Fri, 8 Aug 2008 + 07:40:46 -0700 (PDT) +Received: by 10.100.122.8 with SMTP id u8mr3824321anc.77.1218206446062; Fri, + 08 Aug 2008 07:40:46 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d35si2718351and.38.2008.08.08.07.40.45; Fri, 08 Aug 2008 + 07:40:46 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:47349 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KRT93-0006Po-A3 for + xxxx.klub@gmail.com; Fri, 08 Aug 2008 10:40:45 -0400 +Path: news.stanford.edu!headwall.stanford.edu!newshub.sdsu.edu!news-out.readnews.com!news-xxxfer.readnews.com!panix!not-for-mail +From: anon@example.com +Newsgroups: gnu.emacs.help +Date: Fri, 08 Aug 2008 10:07:30 -0400 +Organization: PANIX Public Access Internet and UNIX, NYC +Message-ID: <uwsireh25.fsf@one.dot.net> +References: <mailman.15123.1216681940.18990.help-gnu-emacs@gnu.org> + <mailman.15143.1216715014.18990.help-gnu-emacs@gnu.org> + <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> + <200808062238.15634.juanma_bellon@yahoo.es> + <mailman.15958.1218056266.18990.help-gnu-emacs@gnu.org> +NNTP-Posting-Host: panix5.panix.com +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +X-Trace: reader1.panix.com 1218204439 22850 166.84.1.5 (8 Aug 2008 14:07:19 + GMT) +X-Complaints-To: abuse@panix.com +NNTP-Posting-Date: Fri, 8 Aug 2008 14:07:19 +0000 (UTC) +User-Agent: Gnus/5.11 (Gnus v5.11) Emacs/22.2 (windows-nt) +Cancel-Lock: sha1:Ckkp5oJPIMuAVgEHGnS/9MkZsEs= +Xref: news.stanford.edu gnu.emacs.help:160963 +To: help-gnu-emacs@gnu.org +Subject: Re: basic question: going back to dired +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 710 +Lines: 27 + +I seem to remember from my early school days it was a campaign slogan +for someone nick-named Kinderhook that went something like + +Old Kinderhook is OK + +- Chris + +"Juanma Barranquero" <lekktu@gmail.com> writes: + +> On Wed, Aug 6, 2008 at 22:38, Juanma <juanma_bellon@yahoo.es> wrote: +> +>> For all I know, it comes from "0 Knock-outs" (from USA civil war times, +>> IIRC), i.e., all went really well. +> +> See http://en.wikipedia.org/wiki/Okay#Etymology +> +> "0 knock-outs" is among the "Improbable or refuted etymologies". +> +> Juanma +> +> + +-- + (. .) + =ooO=(_)=Ooo===================================== + Chris McMahan | first_initiallastname@one.dot.net + ================================================= + diff --git a/testdata/testdir/new/1220863087.12663_9.mindcrime b/testdata/testdir/new/1220863087.12663_9.mindcrime new file mode 100644 index 0000000..734ee35 --- /dev/null +++ b/testdata/testdir/new/1220863087.12663_9.mindcrime @@ -0,0 +1,209 @@ +Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-1.2 required=3.0 tests=BAYES_00,HTML_MESSAGE, + MIME_QP_LONG_LINE autolearn=no version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 4E3CF6963B + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:37 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Mon, 04 Aug 2008 21:49:37 +0300 (EEST) +Received: by 10.142.51.12 with SMTP id y12cs94317wfy; Mon, 4 Aug 2008 05:48:28 + -0700 (PDT) +Received: by 10.150.152.17 with SMTP id z17mr1245909ybd.194.1217854107583; + Mon, 04 Aug 2008 05:48:27 -0700 (PDT) +Received: from sqlite.org (sqlite.org [67.18.92.124]) by mx.google.com with + ESMTP id 9si6334793yws.5.2008.08.04.05.47.57; Mon, 04 Aug 2008 05:48:27 -0700 + (PDT) +Received-SPF: pass (google.com: best guess record for domain of + sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as permitted sender) + client-ip=67.18.92.124; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record + for domain of sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as + permitted sender) smtp.mail=sqlite-dev-bounces@sqlite.org +Received: from sqlite.org (localhost [127.0.0.1]) by sqlite.org (Postfix) with + ESMTP id 4FBC111C6F; Mon, 4 Aug 2008 08:47:54 -0400 (EDT) +X-Original-To: sqlite-dev@sqlite.org +Delivered-To: sqlite-dev@sqlite.org +Received: from cpsmtpo-eml02.kpnxchange.com (cpsmtpo-eml02.kpnxchange.com + [213.75.38.151]) by sqlite.org (Postfix) with ESMTP id AA4F111C10 for + <sqlite-dev@sqlite.org>; Mon, 4 Aug 2008 08:47:51 -0400 (EDT) +Received: from hpsmtp-eml21.kpnxchange.com ([213.75.38.121]) by + cpsmtpo-eml02.kpnxchange.com with Microsoft SMTPSVC(6.0.3790.1830); Mon, 4 + Aug 2008 14:47:50 +0200 +Received: from cpbrm-eml13.kpnsp.local ([195.121.247.250]) by + hpsmtp-eml21.kpnxchange.com with Microsoft SMTPSVC(6.0.3790.1830); Mon, 4 + Aug 2008 14:47:50 +0200 +Received: from hpsmtp-eml30.kpnxchange.com ([10.94.53.250]) by + cpbrm-eml13.kpnsp.local with Microsoft SMTPSVC(6.0.3790.1830); Mon, 4 Aug + 2008 14:47:50 +0200 +Received: from localhost ([10.94.53.250]) by hpsmtp-eml30.kpnxchange.com with + Microsoft SMTPSVC(6.0.3790.1830); Mon, 4 Aug 2008 14:47:49 +0200 +Content-class: urn:content-classes:message +MIME-Version: 1.0 +X-MimeOLE: Produced By Microsoft Exchange V6.5 +Date: Mon, 4 Aug 2008 14:46:06 +0200 +Message-ID: <F687EC042917A94E8BB4B0902946453AE17D6C@CPEXBE-EML18.kpnsp.local> +X-MS-Has-Attach: +X-MS-TNEF-Correlator: +Thread-Topic: [sqlite-dev] VM optimization inside sqlite3VdbeExec +Thread-Index: Acj2FjkWvteFtLHTTYeVz4ES7E2ggAAGRxeI +References: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: anon@example.com +To: <sqlite-dev@sqlite.org> +X-OriginalArrivalTime: 04 Aug 2008 12:47:49.0650 (UTC) + FILETIME=[4D577720:01C8F630] +Subject: Re: [sqlite-dev] VM optimization inside sqlite3VdbeExec +X-BeenThere: sqlite-dev@sqlite.org +X-Mailman-Version: 2.1.9 +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +List-Unsubscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=unsubscribe> +List-Archive: <http://sqlite.org:8080/cgi-bin/mailman/private/sqlite-dev> +List-Post: <mailto:sqlite-dev@sqlite.org> +List-Help: <mailto:sqlite-dev-request@sqlite.org?subject=help> +List-Subscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=subscribe> +Content-Type: multipart/mixed; boundary="===============1911358387==" +Mime-version: 1.0 +Sender: sqlite-dev-bounces@sqlite.org +Errors-To: sqlite-dev-bounces@sqlite.org +Content-Length: 5318 + +This is a multi-part message in MIME format. + +--===============1911358387== +Content-class: urn:content-classes:message +Content-Type: multipart/alternative; + boundary="----_=_NextPart_001_01C8F630.0FC2EC1E" + +This is a multi-part message in MIME format. + +------_=_NextPart_001_01C8F630.0FC2EC1E +Content-Type: text/plain; + charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +Actually, almost every C compiler will already do what you suggest: if = +the range of case labels is compact, the switch will be compiled using a = +jump table. Only if the range is limited and/or sparse other techniques = +will be used, such as linear search and binary search. +=20 +I'm pretty sure, if you perform the tests suggested by Mihai, that you = +will find zero performance difference, neither better, nor worse. +=20 +Paul +=20 +________________________________ + +From: anon@example.com +Sent: Mon 8/4/2008 11:40 AM +To: sqlite-dev@sqlite.org +Subject: [sqlite-dev] VM optimization inside sqlite3VdbeExec + + + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the=20 +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html = +<http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html>=20 + +With a properly defined "instructions" array, instead of the switch=20 +statement you can use something like: +goto * instructions[pOp->opcode]; +--- +Marco Bambini +http://www.sqlabs.net <http://www.sqlabs.net/>=20 +http://www.sqlabs.net/blog/ <http://www.sqlabs.net/blog/>=20 +http://www.sqlabs.net/realsqlserver/ = +<http://www.sqlabs.net/realsqlserver/>=20 + + + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev = +<http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>=20 + + + +------_=_NextPart_001_01C8F630.0FC2EC1E +Content-Type: text/html; + charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +<HTML dir=3Dltr><HEAD><TITLE>[sqlite-dev] VM optimization inside = +sqlite3VdbeExec=0A= +=0A= +=0A= +=0A= +
=0A= +
Actually, = +almost every C compiler will already do what you suggest: if the range = +of case labels is compact, the switch will be compiled using a jump = +table. Only if the range is limited and/or sparse other techniques will = +be used, such as linear search and binary search.
=0A= +
 
=0A= +
I'm pretty sure, if you = +perform the tests suggested by Mihai, that you will find zero = +performance difference, neither better, nor worse.
=0A= +
 
=0A= +
Paul
=0A= +
 
=0A= +
=0A= +
=0A= +
=0A= +
From: = +sqlite-dev-bounces@sqlite.org on behalf of Marco Bambini
Sent: = +Mon 8/4/2008 11:40 AM
To: = +sqlite-dev@sqlite.org
Subject: [sqlite-dev] VM optimization = +inside sqlite3VdbeExec

=0A= +
=0A= +

Inside sqlite3VdbeExec there is a very = +big switch statement.
In order to increase performance with few = +modifications to the 
original code, why not use this technique = +?
= +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html<= +/FONT>

With a properly defined = +"instructions" array, instead of the switch 
statement you can = +use something like:
goto * = +instructions[pOp->opcode];
---
Marco Bambini
http://www.sqlabs.net
http://www.sqlabs.net/blog/
http://www.sqlabs.net/realsqlserver/



<= +FONT face=3DArial = +size=3D2>_______________________________________________
sqlite-dev = +mailing list
sqlite-dev@sqlite.org
http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev

+------_=_NextPart_001_01C8F630.0FC2EC1E-- + + +--===============1911358387== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev + +--===============1911358387==-- + diff --git a/testdata/testdir/tmp/1220863087.12663.ignore b/testdata/testdir/tmp/1220863087.12663.ignore new file mode 100644 index 0000000..588ace1 --- /dev/null +++ b/testdata/testdir/tmp/1220863087.12663.ignore @@ -0,0 +1,98 @@ +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-3.6 required=3.0 tests=BAYES_00,RCVD_IN_DNSWL_LOW, + SPF_PASS autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id D68E769CB5 + for ; Fri, 8 Aug 2008 20:56:25 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for (single-drop); Fri, 08 Aug 2008 20:56:25 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs71287wfh; Fri, 8 Aug 2008 + 07:40:46 -0700 (PDT) +Received: by 10.100.122.8 with SMTP id u8mr3824321anc.77.1218206446062; Fri, + 08 Aug 2008 07:40:46 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d35si2718351and.38.2008.08.08.07.40.45; Fri, 08 Aug 2008 + 07:40:46 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:47349 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KRT93-0006Po-A3 for + xxxx.klub@gmail.com; Fri, 08 Aug 2008 10:40:45 -0400 +Path: news.stanford.edu!headwall.stanford.edu!newshub.sdsu.edu!news-out.readnews.com!news-xxxfer.readnews.com!panix!not-for-mail +From: anon@example.com +Newsgroups: gnu.emacs.help +Date: Fri, 08 Aug 2008 10:07:30 -0400 +Organization: PANIX Public Access Internet and UNIX, NYC +Message-ID: +References: + + <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> + <200808062238.15634.juanma_bellon@yahoo.es> + +NNTP-Posting-Host: panix5.panix.com +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +X-Trace: reader1.panix.com 1218204439 22850 166.84.1.5 (8 Aug 2008 14:07:19 + GMT) +X-Complaints-To: abuse@panix.com +NNTP-Posting-Date: Fri, 8 Aug 2008 14:07:19 +0000 (UTC) +User-Agent: Gnus/5.11 (Gnus v5.11) Emacs/22.2 (windows-nt) +Cancel-Lock: sha1:Ckkp5oJPIMuAVgEHGnS/9MkZsEs= +Xref: news.stanford.edu gnu.emacs.help:160963 +To: help-gnu-emacs@gnu.org +Subject: Re: basic question: going back to dired +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 710 +Lines: 27 + +I seem to remember from my early school days it was a campaign slogan +for someone nick-named Kinderhook that went something like + +Old Kinderhook is OK + +- Chris + +"Juanma Barranquero" writes: + +> On Wed, Aug 6, 2008 at 22:38, Juanma wrote: +> +>> For all I know, it comes from "0 Knock-outs" (from USA civil war times, +>> IIRC), i.e., all went really well. +> +> See http://en.wikipedia.org/wiki/Okay#Etymology +> +> "0 knock-outs" is among the "Improbable or refuted etymologies". +> +> Juanma +> +> + +-- + (. .) + =ooO=(_)=Ooo===================================== + Chris McMahan | first_initiallastname@one.dot.net + ================================================= + diff --git a/testdata/testdir2/Foo/cur/arto.eml b/testdata/testdir2/Foo/cur/arto.eml new file mode 100644 index 0000000..ffa0526 --- /dev/null +++ b/testdata/testdir2/Foo/cur/arto.eml @@ -0,0 +1,448 @@ +Return-Path: <> +X-Original-To: f00f@localhost +Delivered-To: f00f@localhost +Received: from puppet (puppet [127.0.0.1]) + by f00fmachines.nl (Postfix) with ESMTP id A534D39C7F1 + for ; Mon, 23 May 2011 20:30:05 +0300 (EEST) +Delivered-To: diggler@gmail.com +Received: from ew-in-f109.1e100.net [174.15.27.101] + by puppet with POP3 (fetchmail-6.3.18) + for (single-drop); Mon, 23 May 2011 20:30:05 +0300 (EEST) +Received: by 10.142.147.13 with SMTP id u13cs87252wfd; + Mon, 23 May 2011 01:54:10 -0700 (PDT) +Received: by 10.204.7.74 with SMTP id c10mr1984197bkc.104.1306140849326; + Mon, 23 May 2011 01:54:09 -0700 (PDT) +Received: from MTX4.mbn1.net (mtx4.mbn1.net [213.188.129.252]) + by mx.google.com with ESMTP id e6si18117551bkw.39.2011.05.23.01.54.07; + Mon, 23 May 2011 01:54:08 -0700 (PDT) +Received-SPF: pass (google.com: best guess record for domain of MTX4.mbn1.net designates 213.188.129.252 as permitted sender) client-ip=213.188.129.252; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of MTX4.mbn1.net designates 213.188.129.252 as permitted sender) smtp.mail= +Resent-From: +X-Default-Received-SPF: pass (skip=forwardok (res=PASS)) x-ip-name=192.168.10.123; +From: ArtOlive +To: "f00f@f00fmachines.nl" +Reply-To: +Date: Mon, 23 May 2011 10:53:45 +0200 +Subject: NIEUWSBRIEF ART OLIVE | juni exposite in galerie ArtOlive +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d" +X-Mailer: aspNetEmail ver 3.5.2.10 +X-Sender: 87.93.13.24 +X-RemoteIP: 87.93.13.24 +Originating-IP: 87.93.13.24 +X-MAILINGLIJST-ID: <10374608.109906.11909.2011523105345.MSGID@mailinglijst.nl> +Message-ID: <10374608.109906.11909.2011523105345.MSGID@mailinglijst.nl> +X-Authenticated-User: guest@mailinglijst.eu +X-STA-Metric: 0 (engine=030) +X-STA-NotSpam: geinformeerd vormen spec:usig:3.8.2 twee samen +X-STA-Spam: 2011 &bull • e-mailing subject:juni +X-BTI-AntiSpam: score:0,sta:0/030,dnsbl:passed,sw:passed,bsn:10/passed,spf:off,bsctr:passed/1,dk:off,pbmf:none,ipr:0/3,trusted:no,ts:no,bs:no,ubl:passed +X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply +Resent-Message-Id: <19740414233016.EB6835A132F5FCF4@MTX4.mbn1.net> +Resent-Date: Mon, 23 May 2011 10:54:07 +0200 (CEST) + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/plain; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +ART-O-NEWS; juni 2011 Westergasfabriekterrein Polonceaukade 17 10= +14 DA Amsterdam tel: 020-6758504 info@artolive.nl www.artolive.nlJuni= + expositie bij ArtOlive: Peter van den Akker en Marinel Vieleers + + +Zondag 5 juni + Elke maand vindt er in de galerie van ArtOlive een expositie plaats. We = +lichten enkele kunstenaars uit (die je misschien al kent van onze website= +), waarbij we een spannende mix van materiaal en techniek presenteren. Ti= +jdens de expositie staan we klaar om elke vraag over ons kunstaanbod te b= +eantwoorden.=20 + +De exposities zijn te bezoeken van maandag t/m vrijdag, tussen 10:00 en 1= +7:00 uur. De opening is altijd op de eerste zondag van de maand. Dit valt= + samen met de Sunday Market die elke maand op het Cultuurpark Westergasfa= +briek georganiseerd wordt. De Sunday Market is gratis te bezoeken en staa= +t vol met kunst, design, mode en heerlijke hapjes, en er hangt altijd een= + vrolijke sfeer. Een ideaal moment dus om in te haken en deze maand twee = +kunstenaars te presenteren: Peter van den Akker en Marinel Vieleers. + +We verwelkomen je graag op zondag 5 juni 2011, van 12:00 t/m 17:00 uur op= + de Polonceaukade 17 van het Cultuurpark Westergasfabriek in Amsterdam!=20= + + + + bekijk meer werk op www.artolive.nl... Peter van den Akker + + +"In mijn beelden en schilderijen staat het mensbeeld centraal; niet als i= +ndividu maar als universele gestalte, waarbij ik op transparante wijze ti= +jdsbeelden en gelaagdheid in het menselijke handelen naar voren breng. Ve= +rhoudingen tussen mensen, verschuivingen in wereldculturen en verandering= +en in techniek, architectuur, natuur en mensbeeld vormen mijn inspiratieb= +ronnen. Het zijn allemaal beelden en sferen die naast en met elkaar besta= +an. Mijn werkwijze omvat vele technieken in verschillende materialen: sch= +ilderijen, gemengde technieken op papier/collages, zeefdrukken, beelden i= +n cortenstaal, keramische objecten." + +Peter van den Akker exposeert regelmatig in binnen- en buitenland bij gal= +erie=EBn en musea en is in verschillende kunstinstellingen en bedrijfscol= +lecties opgenomen. + + + lees meer over Peter... Marinel Vieleers + + +Marinel Vieleers probeert het menselijke in de bouwwerken - en ook vaak i= +ets van het karakter van de bouwer of bewoner - te laten zien. Het zijn m= +aar subtiele details die dat alles weergeven. + +De 'tand des tijds' of invloed van mensen op de gebouwen spelen vaak mee = +in het werk. Koper, cement, lood en andere materialen worden in haar nieu= +we werk gebruikt. Op deze manier kan ze gemakkelijker improviseren en nog= + directer op haar gevoel afgaan. + +Marinel is gefascineerd door de schoonheid van het imperfecte. De gelaagd= +heid van ouderdom, het verval. De imperfectie die ontstaat door toevallig= +e omstandigheden maakt een huis, een muur, een schutting, hout of steen b= +oeiend. Het is doorleefd en het toont een stukje van zijn geschiedenis. + + + lees meer over Marinel... =20 +ZONDAG 5 MEI - Juni expositie in de galerie van ArtOlive met Marinel Viel= +eers en Peter van den Akker + +Opening op zondag 5 mei, tijdens de Sunday Market op het Cultuurpark West= +ergasfabriek in Amsterdam. Je bent van harte uitgenodigd om tussen 12:00 = +en 17:00 uur langs te komen! + +Daarna is de expositie te zien op werkdagen (ma - vrij) tussen 10:00 en 1= +7:00. De expositie duurt tot 24 juni 2011. + wil je niet langer door artolive ge=EFnformeerd worden? Klik dan hier om= + je af te melden.=20 + kreeg je dit mailtje doorgestuurd en wil je voortaan zelf ook graag de n= +ieuwsbrief ontvangen?=20 + klik dan hier om je aan te melden.=20 + +Deze e-mailing is verzorgd met MailingLijst + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/html; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + + + + + + Artolive + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
3D"artolive"
ART-O-NEWS • juni 2011 +
Westergasfabriekterrein   Polonceau= +kade 17 1014 DA Amsterdam   tel: 020-6758504  info@artolive.nl<= +/a>  www.artolive.nl
+ + + + + + + +
Juni= + expositie bij ArtOlive: Peter van den Akker en Marinel Vieleers +

Zondag 5 juni
+ Elke maand vindt er in de galerie van Art= +Olive een expositie plaats. We lichten enkele kunstenaars uit (die je mis= +schien al kent van onze website), waarbij we een spannende mix van materi= +aal en techniek presenteren. Tijdens de expositie staan we klaar om elke = +vraag over ons kunstaanbod te beantwoorden.

+

De exposities zijn te bezoeken van maa= +ndag t/m vrijdag, tussen 10:00 en 17:00 uur. De opening is altijd op de e= +erste zondag van de maand. Dit valt samen met de Sunday Market die elke m= +aand op het Cultuurpark Westergasfabriek georganiseerd wordt. De Sunday M= +arket is gratis te bezoeken en staat vol met kunst, design, mode en heerl= +ijke hapjes, en er hangt altijd een vrolijke sfeer. Een ideaal moment dus= + om in te haken en deze maand twee kunstenaars te presenteren: Peter van = +den Akker en Marinel Vieleers.

+

We verwelkomen je graag op zondag 5 ju= +ni 2011, van 12:00 t/m 17:00 uur op de Polonceaukade 17 van het Cultuurpa= +rk Westergasfabriek in Amsterdam!

+
+

3D""  = +bekijk= + meer werk op www.artolive.nl...   

+
+
+ + + + = + + + + + + +
3D""
+ + + + + + + + + + +
<= +a target=3D"_blank" href=3D"http://www.mailinglijst.eu/redirect.aspx?l=3D= +154043&a=3D10374608&t=3DH"> <= +/td> + Peter van den Akker
+

"In mijn beelden en schild= +erijen staat het mensbeeld centraal; niet als individu maar als universel= +e gestalte, waarbij ik op transparante wijze tijdsbeelden en gelaagdheid = +in het menselijke handelen naar voren breng. Verhoudingen tussen mensen, = +verschuivingen in wereldculturen en veranderingen in techniek, architectu= +ur, natuur en mensbeeld vormen mijn inspiratiebronnen. Het zijn allemaal = +beelden en sferen die naast en met elkaar bestaan. Mijn werkwijze omvat v= +ele technieken in verschillende materialen: schilderijen, gemengde techni= +eken op papier/collages, zeefdrukken, beelden in cortenstaal, keramische = +objecten.”

+

Peter van den Akker expose= +ert regelmatig in binnen- en buitenland bij galerieën en musea en is= + in verschillende kunstinstellingen en bedrijfscollecties opgenomen.

+
+

3D""  = +lees meer over Peter...   

+
= + Marinel Vieleers
+

Marinel Vieleers probeert = +het menselijke in de bouwwerken - en ook vaak iets van het karakter van d= +e bouwer of bewoner - te laten zien. Het zijn maar subtiele details die d= +at alles weergeven.

+

De ‘tand des tijds&r= +squo; of invloed van mensen op de gebouwen spelen vaak mee in het werk. K= +oper, cement, lood en andere materialen worden in haar nieuwe werk gebrui= +kt. Op deze manier kan ze gemakkelijker improviseren en nog directer op h= +aar gevoel afgaan.

+

Marinel is gefascineerd do= +or de schoonheid van het imperfecte. De gelaagdheid van ouderdom, het ver= +val. De imperfectie die ontstaat door toevallige omstandigheden maakt een= + huis, een muur, een schutting, hout of steen boeiend. Het is doorleefd e= +n het toont een stukje van zijn geschiedenis.

+
+

3D""  = +lees meer ov= +er Marinel...   

+
+
+
+ + + + + + + + + +
3D""
+ + + + + + + + + + + +
+
+
+ + + + + + + + + +
3D""
+ + + + + + + + + + + + + + + +

+ 		ZONDAG 5 MEI - Juni expositie in de galerie van ArtOlive met = +Marinel Vieleers en Peter van den Akker
+

+ 		Opening op zondag 5 mei, = +tijdens de Sunday Market op het Cultuurpark Westergasfabriek in Amsterdam= +. Je bent van harte uitgenodigd om tussen 12:00 en 17:00 uur langs te kom= +en!
+

+ 		Daarna is de expositie te zien op werkdagen (ma - vrij= +) tussen 10:00 en 17:00. De expositie duurt tot 24 juni 2011.
+
+
+
3D"Kunst wil je niet langer door artolive geïnformeerd worden? Klik= + dan hier om je af te melden.
+ kreeg je dit mailtje doorgestuurd en wil je voortaan = +zelf ook graag de nieuwsbrief ontvangen?
+ klik dan hier om= + je aan te melden.
+

<= +HR SIZE=3D1 STYLE=3D"COLOR:#d3d3d3" SIZE=3D1>Deze e-mailing is verzorgd m= +et MailingLijst

+ + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d-- diff --git a/testdata/testdir2/Foo/cur/fraiche.eml b/testdata/testdir2/Foo/cur/fraiche.eml new file mode 100644 index 0000000..c0bf442 --- /dev/null +++ b/testdata/testdir2/Foo/cur/fraiche.eml @@ -0,0 +1,10 @@ +From: Sender +To: Recip +Subject: search accents +Date: 2012-12-08 00:48 +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit + +line 1: Глокая куздра штеко будланула бокра и курдячит бокрёнка +line 2: crème fraîche diff --git a/testdata/testdir2/Foo/cur/mail5 b/testdata/testdir2/Foo/cur/mail5 new file mode 100644 index 0000000..b72195d --- /dev/null +++ b/testdata/testdir2/Foo/cur/mail5 @@ -0,0 +1,625 @@ +From: Sitting Bull +To: George Custer +Subject: pics for you +Mail-Reply-To: djcb@djcbsoftware.nl +User-Agent: Hunkpapa/2.15.9 (Almost Unreal) +Message-Id: CAHSaMxZ9rk5ASjqsbXizjTQuSk583=M6TORHz=bfogtmbGGs5A@mail.gmail.com +Fcc: .sent +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: multipart/mixed; + boundary="Multipart_Sun_Oct_17_10:37:40_2010-1" + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: text/plain; charset=US-ASCII + +Dude! Here are some pics! + + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: image/jpeg +Content-Disposition: inline; filename="sittingbull.jpg" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/4QvoRXhpZgAASUkqAAgAAAAIABIBCQABAAAAAQAAABoBCQAB +AAAASAAAABsBCQABAAAASAAAACgBCQABAAAAAgAAADEBAgAOAAAAbgAAADIBAgAUAAAAfAAAABMC +CQABAAAAAQAAAGmHBAABAAAAkAAAAN4AAABndGh1bWIgMi4xMS4zADIwMTA6MTA6MTcgMTA6MzM6 +MzcABgAAkAcABAAAADAyMjEBkQcABAAAAAECAwAAoAcABAAAADAxMDABoAkAAQAAAAEAAAACoAkA +AQAAAMgAAAADoAkAAQAAAGsBAAAAAAAABgADAQMAAQAAAAYAAAAaAQkAAQAAAEgAAAAbAQkAAQAA +AEgAAAAoAQkAAQAAAAIAAAABAgQAAQAAACwBAAACAgQAAQAAALMKAAAAAAAA/9j/4AAQSkZJRgAB +AQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwc +KDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIy +MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCACAAEcDASIAAhEBAxEB/8QAHwAA +AQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIh +MUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpT +VFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5 +usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAA +AAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEI +FEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVm +Z2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK +0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDq77xdrX/CQ6laRXjRxQTF +ECovA/EUg8Sa6W/5CUuP9xP8K5yWQnxjrw9Lwj9BWjkgZHFAG6mu6yV51OXP+4n/AMTUq61rBB/4 +mU2f9xP/AImsJJTuAJFW0YDnfmgCTUPFGqWFq882p3G1eyqmT/47VfRfGGpawkgGp3CyIeg2cj1+ +7XK+O7zybCGMNjzHyR6gD/69ZvgG8zqU67vvRZH4EUAesJe6m/XVLv8ANf8A4mpf7Qvl/wCX+6b6 +uP8ACs+ObKdaeh3Hg9aANTw/4gurjxTLpU7tIv2cTKzHpgkH+n5UVheHGI+KWzJwdNP/AKFRQBzD +7f8AhMfEDEHH24j/AMdWrs0oCkDrVKJs+NfEsZ+79u/9kWrd5GqKTmgCstwwkyT0p5uzu61mOzbj +zSFn3DmgDB8ePLPe2MEQZyykhRzk9/5Va8D6Vd2Mz3d3CYxJHiPd16+la0hhMybkUvxhj1HWr5uM +uB0wMYoA3YJARjvV+DBPasC2lYsOuK3LVunWgCLQRj4sIPXTGP8A4/RSaF/yV2P30tv/AEOigDmY +QD408UE9ftw/9AFXpv3iFT9Kgs4t3jXxV6C+H/oAq5cxkMcCgDFltXVyMVVv7iGwtzNcNsQfiT+F +a8jbAxdgFAzmuTZZfEV81vG+xTyX/uIPT3P9aAIBr9vdNHcQI/lxk5DDBrfsLuK+jE0MqupPOOx9 +KzNY8L6fbaaYrdGXb3BOcnuT+FcpodzN4c8RRRyylrW4IDE9MdM/UUAes2wbOAK27PhRms6CJlwc +VrWowRkUAV9CP/F3YffSm/8AQ6Kfo+P+FuWp9dLf/wBDooAxrH/kd/Ff/X8P/Ra1evUOcgVW01Qf +G/izIz/py/8Aota2LqPK4xQBxniWc2mi3MxBGFA/Mgf1rmtEF/Z6HNqMNuzvPnY+7G1V6Hoe+T0r +qfGmnT3Xhm8WNWJVQ+B/skH+lUPBt3d3PhuzXyBM6xBY0YfKDnALewxmgDE1BfEDaPaXNzMRJPIQ ++TjCgDHb69u9ZGt2Us2lrdNDtMLAgq27Kng84Fd74qnaMwWB8qWRTnzUcfePGSOx4ziuf1kzT6S9 +tuRHlVUG5sDJOMA+lAHofh5/tvh3T7k4ZnhXcfcDB/UVuRQEdqzvDelPo/hywsJGDSRRjeR0yeTj +2ya3I8/3aAMXSU2/FmzJ/wCgbJ/6FRUunf8AJV7H/sGy/wDoQooAyNJXf448XYPS+X/0Wtb8ynyj +0rm/DIll8W+KDKQ0pvF3FehPlr0rvINMzbfN8rsc7upH0oA5ie3mktZSI1ICn5W43e1ec6ZrDwax +facIj9liUNtUcgE8j0IzXrHiqS20rQJbiadoyBsWQjc2T2HvXnvhbREuzeXTbvMlfILcsF6D6jFA +GJr+pWE1ymFkwFzhlwo+i1xevazLd3Fva2+UiQhh7kdPyr0jVfA8t0BeXNybe35UK2EJAJwST/QG +uS1Pw7HYalbKHUIxYxyDd8wHUnNAHsnhXVBrGhWkrBlmEYVww6sAATXQInA5rn/AOZtIa3mQHZI+ +xwfvAnJ6d8n9a6yazEKhlzgUAc1YAr8WbH302X/0IUU6xBPxYsSe2my/+hUUAV/Bdj5fi3xWJJDJ +JHeopY8bj5a5OK9AUArwARXEeFjjxh4xbub5f/RYrsIZgJhGTjcuQMGgDnfHiwnw1KJoVkUuB8yg +hfeuZ+HemTLpjx3OCZNzKUbPy54/Sut8Z263OlJE1wYgzkkjvgH86yfBb+XYWuIGiEithWzn9aAN +loTcO0ctuGjV9oMg5JGCSOOnp9K8/wDH1qH1iERrukRAqqB3Jzj9BXpsk6F+oyCuRjJ54rhNcg+3 +Ge5XiUSL5ZGc87sdPagDQ+HlvJHoAdo9h85mUY7dK7WSRCoB6HiuV8IiW10JYs7yszDJ7fN/k1tG +Rpb4xj7qpnj3Iwfx5oAwLMgfF+1UHI/suTH/AH3RTLJNnxltx2Olvj/vuigB3hgf8Vp4vH/T8v8A +6AK6aRWFk2CA2CPSua8M4/4T3xcp/wCftD/45XR6q32e1JjUySCRdqA4J3HH9aAKHiJTceH4mliK +r5e5lDfMpx2Iqp4eQR6Zp75Y4jX7xyfTn8q29djjbS/LMqxYGFdugNZWlskOh2pKgYj2AqO4OB/M +0AW7+NLQ3Fwi/O6hsk5yRwOO3WuS1qGJtNuvN3iNJkX5e+EIxn8f1re1e4ubq8jSOMiBArZJ/wBY +xOcfQcZ+tVNTsYh4dnjmG9PMJIP8XYUAQ20z2Hg6OeJGTYQzd+N3Le+RzXQ6TGwtjLLkuxAy3XAH +f8Saw9Mlt7vwsI4yZI9m07xtyM/y5rqodqxIFAIx1oA5iDj4w2ZHfS3/APQjRSw8/GGzx20x/wD0 +I0UAee+I/GV/4S+IXiAWlvFKJ7gM28njC+1Ubn4v6xclC1hbAq6vwzdjn+lXviB4X1O88b6lPBYX +EkUkgZXWJiDwPQVzH/CH61zjSbwj1EDf4UAbV78YdZvYPJbT7UA+7HP61HbfFXXLW3SFdOtSqZ67 +v8fesg+Ddbzn+yL3P/XBv8Kcvg3Xc5Oj3x/7YP8A4UAaY+KuvIQP7PtM5JXKt/jUF78Udcu7F7WS +ytEVv4grZHPB61VPg/Ws/wDIGvs9v3T/AOFMPg7XcHOk32P+uD/4UAWLb4l6vb2zQJZ2m1gP4WGC +FAz19q17f4va0sSobS04GB8rf41z3/CIayOuk3g/7d2/wqRfCWr8f8S27/78P/hQB33w78Q3fib4 +jR3l3HHG6WTxgR5xjOe/1oq78JvCmo6dq8+qXUBhhETQqJAVYsSDkA8496KAP//ZAP/bAEMABQME +BAQDBQQEBAUFBQYHDAgHBwcHDwsLCQwRDxISEQ8RERMWHBcTFBoVEREYIRgaHR0fHx8TFyIkIh4k +HB4fHv/bAEMBBQUFBwYHDggIDh4UERQeHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh4eHh4eHh4eHh4eHv/AABEIAWsAyAMBIgACEQEDEQH/xAAdAAABBAMBAQAAAAAAAAAAAAAH +AwQFBgACCAEJ/8QAVhAAAQMCBAIHAwYHCwoGAgMAAQIDEQAEBQYSITFBBwgTIlFhcRSBkRUjMqGx +0RYXQoKSssElJjNDRFJicpOi8CQ0NmNzo7PC0uEnNVNUVYMJRWR08f/EABQBAQAAAAAAAAAAAAAA +AAAAAAD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwDqjNudMEyw6lvE3HEqUnUS +kDSmeEkkCTB241Aq6X8mgSL5J/PT99DPrX3S232GwohPbp5/6v8A70PMsrSuwYKgkgtjiONB0Ovp +lykFQLhJ8+0Fefjoyh/6/wDfFBctsJWkpt0wRzSK9ft7cwVMNmf6IoDOOmjJ5/lB/SFejpmyeT/n +I/TFBdFqwACGGR6IFOWbRhZ3Ya349wUBgT0y5QP8qT+n/wBq3T0x5OPG8A/OFCy3tbZCRpt2j+YK +cotLMjvWjM/1BQEs9MWTR/LUn0VWfjiyb/72hki1sw5q9kaHL6ApZDFspQHsze3DuCgI6umLJqUg ++2Eg+AJ/ZSR6aMmj+UOfon7qozjTACUhlsCP5opo402AQWkeumgIK+mrKAA0uuK8e6R+ykj03ZU5 +JePx+6h6gaTGlPHmkUwzDjlrhDKVvtIWXVaUIjj4+6gKKOm7KZUQrtU+YCj/AMteHpuyvvpbeV7l +f9NAHEc3Ls8ed9iYZLTZIAI4+vpRAy1i7WKYS3dhtIDghQHIjiKC9npwy3O1rcH3K/6a1V034Ef4 +OyfPqlf/AE1WG1KKxAAHhFO0LgTCfhQTf47cGPCwuJ/qr/6a9R004as7YXcn0bX/ANNRLbqhsAOF +OWnVgEnh4UEm30wWbhhvBL5e3Jpz/ppwOlMLSC3lzElT4suD/kqHN642YT9tIu3T7h2UYHnQWW36 +S9YleXcQA8Qhf7U05b6SMPXiFpYGwfZfu19myl9XZ6leEkR9dVBy7XEFXLcUN+mPEn7VnCrplZQ6 +zclaFeCgAQfqoOqbO4L6VBxvsnURrRqConcbisqo9FOYE5mw1zGENdkLllpZQSCUkApO/wCbWUAY +63i9N0x3v5SkHy+aqj5QBXhFqRzbFWrrjrWm5a1ER7Unh/saqOSXP3Bst/4pNBbWhLY1nvCt1NyZ +mRWtuoKESCOdKjSOB99BiEjYHYU4ZQpKpTwpArTxpVt8g7RQPmlQkpP2UoFE8VkU0buEkK4A+NbN +rCzHDagfQCmN69TtsPjTRLoBA1b0oHFaZPwoH6IPODFJPwOG9asklJJ228a8egp9aBspcHhvQf6T +b55eZHWVu6m2kgISDsnaT76Kt46hlpx1WyUJKj7q59xzEFXV9dXC1krccK9/M0CybtYUTrMnj50R +uiPGLl69OHBTaWG2isgjdRkffQgFyoqO441eOiW4WM1W8D6aVBXpFAfGCVGQmnDZO07jmaYW74G0 +yKfNuJUBB50D0ISAIAiNyK3UUkQIpDtgUhAAIApNToAKQACedAotaZgCfOvQrUCAIpsVAma31kEE +bbUG60Dwn1oW9O6S3h2HqnYvK291E1SlEkzvQr6f1qRg1gVHc3B4+lAaeq2rXkFhcz3NPwcXWU06 +pzqnchtQ4CkBQjT/AKxVZQDXrpOEXLJO0XYj+xFVDI6lHAbEeLSfsq1dd2UPWyxMG7HHhPZCqtkX +/wAhw9Svo9gj7KC4Wx2MCKdt7ncTTaxbGmBz3p4hISNzHhQJrQSYnjXiUkQOYpZzupJJpFK5g7n3 +0ChQSkbxSluSFDfam5WACokj1NetPAqA29KCXb0HdQ9TSwQlSduVR6XBpJnetkXWgCTQSTQCRpMk +Uk8djzpsm9ExNaOP6pOqBQRWcblNtly/eI3SwoTPiIrnbEXAl7j4UaulrEHLbJ9z2ZEuKS2fQneg +Fevy62ASQYoHCHkqXsqrl0U3aWs52mpelLgU3v5jah20/C9zO/hVmyc8Bj+HjfvXCB9dB09bq0mD +zqQZUkAVDtriOdO7ZzSYO8UEoHBw3r0KB3mm6XUxERWdpO0UDhJBVvSulKh9kU2bUdzvvSzKu9tI +mgXShOjfaTQk6xZ/cOw4GLk7/m0XlqlGxEcCDzoP9Y1KhgNhI/lR/VNAWuqGpJyG3pBHdVPn84qs +rTqgqJyKyCI+bXHp2qqygH/XoQEW9i5M6r0CJ/1VVnJKP3Aw8QP4BvYf1RVh69K1lm0BMgXyQPL5 +moPJ50YDY+TCP1RQW+yRoaC44HhSygNXe99NbW7lCUKA48K3cdgbk8KDa5IAMcKj3XiklIMTSrj0 +zCppi8STwNAo5cEo4RXtq53pn1pke8NIUR5UoydWoA+lBMdrCdjSLr5G8yfCmJdWE6ZpNx2NiZoH +4uZjlThKitJ35TUOl4DntUlZOgoiaAe9Nl+tnDbS1QAULdlzxHgftoMLdm4VzAmKKvTsvsrywUqS +y8y42seBBBB9xoSWTa7hVwocGmyrh7qDZpUKFXzoptG77NdoFkwiXD7uFUvCMOu7+5SxbNLccO8A +TRd6Nsq3eB5mbVezq9l7ThsCTEUBcQvhtuKdWy0qVJ40wS4NqcNOTvA+NBKpMokRSiFKnx8opo07 +KAAYNOGwrko0DlB/JO005Y0gTNNW9pkzTm3SDJJoHIQhIlKZ50IesgFDLdgYgC74/mmjC03KaEXW +WBTlmykH/Ox+qqgJnVAV+8S3hMbOA78fnVVlZ1Oio5BbnkHI/tVVlAOOvSIbtlSN8QSD/Y0yyayh +3Ldg4BBNs37+6KedfERaWqo2+UE/8E1FdGdwF5Yw0kgzatmPzRQWdhnRClCTWPAEHeKkENodbSpO +xpBy3IJJjegjFwjh6U1eWSoxw50+uGQHPE86j7qEqiY9KBstYSe6B61gfLapB5QaaPOQs77V4twR +wNA8TcFyPKsdd25UyS6RBAivXXJSDQOm1jyinlq/CoSY22qDDh1AVI22mUqmNqCpdI+GXOZb9Ng1 +I9jZL+qNiVSAPiKiMhZQFixcHGGgkvJPaJJ4JBolsKaTdO9zdTcKV4CdqalLSrtevcQEkeVA7ytl +/CcOm7sLVKFugb+VSt06k3WgfSSO9HKss30NsagAkAd0GminCXSr8onegfNKSDO/nJp22tCSDUYF +xEJpVDkkwDPrQTTCoGxBmnjDp4KG9QVu4rVxjepS3WY1KVPpQSbSiT508tBvBImo9hJICt96f2yS +TwIigkWht6UHes8ojK9mJE+2Dj/UVRhaEpn7aDnWhH71LP8A/uj9VVATupmZyCiePzn/ABVVledT +OfxfszG6XYj/AGyqygHfX3A+TrPmTiKZ3/1JqvdFX+h+EqPO2TU91+NrK1Mf/sER/YmoLopSpWR8 +II/9sPtNARbNSdAKdzHCtnY0Eq4zypnbFaEAqImnRUFt6p93OgjbyEEn6qhrxQUo+FTGIp7m23rU +HeGKBi6E6orRcRtWyiSoxWp4ERQeoAI3NeLSnnW7Q2341s5skcOMUCSEpMk08to0gA/Gmu4HCvW1 +qoPMQWtOJIDR/I3HiKyycacfU4o86Y4qpxBcutiUphIpLLy1LGtSN1RsTwoLS0SQpx3uoG4HlSHa +gq1JMSa9ullVulJAA5+dN0wYiKCQbcJ+lNLoVzmmjKoEmndudSoVAHpQOrbVq3mIqWtYME7Co9oE +xHAeFPWVqCQY250EsyuBsSPWn7C1CI4+tRdq4F7gjzFSduoCBHoKCRYUYAG1CHrQJjKFqqB/nqf1 +FUWbck96NO3jNCXrPmMm23Ha9R+oqgJPUvM9H6PLtR/vTWVt1MkoT0eMaQAVpdUYPPtSPurKAb9f +ZQNkwnf/AMwR/wAA1GdD7QOQMHUeduPtNSfX3H7n2y4mcSSPT5g1G9D69XR9gxCeFvHHzNBczGkR +tWqXIBjlW4HdO0U1XqQsGOe9ApcAKTHEGoe7tyJqWWSBv61roQ6nvCgrD7cEyPqpNCYn0qavrMhW +w5U1NsYAAoGLAEmaUcAinItSZ8R4VjjJ0Qo+tBHK+NeoI0kVXs3ZptMIS42wk3FwB9EGAPU0KMWz +zma6uCtm67BoHZLSQEx6nc0BdzRfBq0UhKu8BwFI4BchttK1EyrlPCqjhmJOXlrbquHUPOrbBJJ5 +8+FaXC32NTguLi2MmCYU2fLjPxoCel0vDUFFQPCnSAQaH+Vc625fRhuKpbtX+CHAruL+6iC0QtKV +JUFA8CKByyFE8KeI2CTzpG2E8h8acpQSRCRH20DxlRMGPqp2zOkJB2prZtlRIg/Gn7bSUwNW9A+t +BAO4B8Kk7UeJmo20SUkmeHlUnbK2G1A+aiZjfxoS9Z8k5Ltwrleo4f1VUV0d6IG/OhR1mkg5HZkz +/lqP1VUBI6l5H4vmRJOzvHl86aykOpOf3iEb/Sdj+0rKCgdfUj5NtxzGJI9/zBqP6Gkj8XuDDn2H +/Malev6lHybakQD8oNg7f6hVMehVIV0dYMqJhj/mNBcm29Q24U3u2gDtUohvS2dtzTO6H+IoI93d +PmBTcq0xBmnDoM7b7U1KZUQCBvQLKcQtuFRSCwAmBFIvBxJ2MGaVAUUhQIngaBNUg7kVV81YoUhV +rbuweCiOPpUxmC9Rh9it5Su+dkDmTQ9Nz2q1PrO5JCd5JNAxxyztBaOOXyktsNp1rUefqaH7dpc5 +hvlDCLJabVJgOLG5H2D0q7YhhV5nDFrfBLYrTbA9reOAwAkHYep/YKL+X8qWGHYe1aWjCUBtMDQn +c+cmgDtvlq7scEQ2grTdtnVqKdhVXx3EscYTpumoIkdogbKHmDXRWJ2NotlxpbjRUOYVJ9N6G+bc +Mt3EdghaFx9IAUAXfxB25bIe0haOHd+qiR0QZ1dRdIwXFXdbayEsOqP0T/NNU/MuXH7Qret0KKUn +dIE/CoCxUW3UnfcyCOINB2BaiRI8Nop+0gxP21R+iPHF45gCUPq1XVsezd347bGr8wlQERQLWp0r +Gw41IoAKp2k0xQkg/Qnwp9bSYJSZFA7YSfjUhbN6uMAimTBIUCakbZW44UD1hkJAoUdZ9pKMgNqH +H25H6qqLTR7oPOhZ1ngD0dSOIvGz9SqC4dSZJGQyeRU7H6YrK36lBH4vk7flvQfzxWUFE6/oBwm1 +Vz+UWh7+wVTboPSR0cYKkCf8nk/pGpHr/o/e9aLjf5SbH+4XTPoOUk9G2CyCT7PwA/pGgvaR3eAN +M71E8qkkJBRsDvTa7QNBFBCuoHGKaqbGuZEU/udIERvTbUmSNO9A0ukjSTG9Nm1BC5PDnUg8pIHC +ozEn0tMLcCeCSfqoB9nrElXOLi2bJ0IOkeE86rVzcBDKnAoBLRgCNyfL/HjXmO4koPaoPaOEmf5s +86aYHbPY7mmywdk6mmyHbjwKZkz60Bl6NcGRhWX0XDrWm6vPnnIHeAPAe4VZx2iEFSVrKYgpI0/X +UJeYuzhrae0Qt9SANmwdIH7ajh0j4WrW204dYEEE8NqCXW6+7dP2yUMNnTKdZJ24cao2ZGCVO9ol +tzcmUnh/j7qXTm3tbC8xdcKQnsWRJ7pIUoq+qKqeMZ3YUp9xxxsJcUCQngOJ2+NAmptDxKHEqSCN +vGhnnCxOHY8sISEpe76YECecfb76J2GXDV8wLj2a4DDhlDikEA+h8Kr/AEk4St7Chdtd4sHUI4xz +Hw391At0JY38mZntmnHdLV38y4nlP5J/x4102ywVAEbiuLMDfcYeauGjpUlQUDHMGu1MpXPyll+x +vkbh5hKifOKBZtuCRoiKWbQoGYpx2KkmYO/lSiGjI2EeYoMaQSSDTplJA2rxtJG1OmUgD/tQKM6h +50MOsv3ujlwgzF02ftopJMAAxt5ULeslKujq5I/9w2frNBb+pM6Dkfsuep8n9NH31lJ9SUfvOBgb +F/cf10VlBVuv7q/B61g7fKTW3/0Lpn0IjT0d4IN/82HPzNO+v8Vfg/apkR8osn/crpr0HEno5wUE +cLeJnzNARGSCmCD8ab3IBkAcqctjbhwrV1oqTMAUEBdpABnemRAHvqYu7WASQDNMOw7xAKfHjQMn +YjvDaq/m64RaYFevcNLSj9VWp+3IEyKo3S8U2+RcSXIkthI9SQKAKXV4u5ue21BLY3JngImrb0S2 +TrmCXuYVuKb9pfUkr0nZpA4Ty3maHDSXsQLOHW/8PdOJZbHLvH7AAfjXUOW8rWWGZbssMTZtutMt +BGpbQMniVRE7mTQDzMfSJctYXcpwbLq30MoSFP3KinXO3cQBJ9SRQpu/lvE8WS+jCQXXVAyylaUu +T68TXQubsNNpZK1YwLVsDV3mSSPSFCoPJeTH3rv5dvO2UiYtu1RpKxH0o4xvzNBHXuX0WnRpcGzQ +supWlSyqZMjccfGN6FGG2+Iv4u5cW+ALxNLSoaZUfmwoc1Abq9Jrq3GsPtrbI12gcFNHaI3oJW9q +7Z9qbEpSpapg+NA2ezHnJ4rZxLCrZLVshHYWzdutBXPECCoAjzqRtlKxNpxp22U0tSTLS4kCPXen ++XLW5xR2EN3iXE/SCmwUj3wD9tTSsHLPdUtK1cYiaDnq1bXa4jcWLuy23FAe7b7q616v1yb/ACFa +JMyySg78INc19IViLHNAuG0hPa94jzTxo79Vq/DtjiWHDcJWHk+iuXxFAXrls7xO1etJkCCaevNy +NXOvGGxG6QDQaIbM8AactohMRWyWyD605aSInwoGwTIgCD40MOsa3/4cXh8Hm/1qLSkhMqNDHrHN +T0Y38Dg43+sKCd6k4H4EHxDj36yKyvOpTIyZpJPF79dFZQVLr/T+Dtt4DEmf+Cuk+g1uejfAzEk2 +wn4ml+v8EjK9sYknE2Y8vmXKT6BQ4OjTAyT/ACfh+caAiBoDYCaxxOlAGnjSralQJg7V4+sdmBtQ +Rd2gaZM7nwqLdShCzE1NXS2wmJmot5CSdSQregY3KthE+kUNOnjtF5Hf7PVHatlfoFCim+hOkwDI +86p3SRhyr/Kt/apaKipklIHGRuKDmjKV81ZZuwi6cWSlNzpE8BtE/EiuqLbMPtFi2WVDVA2rjXEy +5bX1skApW2AQPA6ia6awJQNvavpSZWgKI5EEUBLwmwbxFSHMQYZd098FSQQD761GMWuK45eYPhYT +cKsUJNw4ncIKphI84BJqs5hzKcPy4+q0lVypISNIkk8gPEkmKnsuZCvMHyEi3sMQFnmC6V7Re3Rb +16nFcUnyTwHp50Epmxm3ssnrYeUe0uEHSIoIYg0cKS9cOW5Uw1ClqHEJ5n1q59JWKYhh7tthN0+u +7et2NetKY1pHExyoft4lfYnir1u6sPWL+mUqT9HxE+HxoLhaXS2sND9k5rbdROpJ2INa212OzK1L +nn3qhMtMPYU67hThKmEElon+YeHw4e6nFxPblQBKPyY/x5UAt6VLwPZoba3DYbn3knf6qLXVGbcV +e4u8QezS02kEjmSTFDHNuXcRxnNDKrG3U4SnRJMSZ/711B0I5OcyhktuxvA37a4suPqRuJ5CecCg +ujoEQfCtmOI2rZ0d2vEpEbcKBy2AVcvfS6UgbDnTNJPnSrRK5UmRQKqC9ESDvyoY9Y4q/FlfiQfn +G/1hRPWogaYihh1joPRhfEj+Mb/WFBMdSon8DgDyL8emtFZWvUnCRlFYSRIL0+utP7IrKCr9f3/R +q2PH90mR/uV170EKKejLAxAP+TbfpGvOv+CMsWpPPEmY/sV1r0Bk/izwPcx7Od/zjQEhlRUNk+6l +XEymY5UmyDp4x50uggpHemgj3mZSSRA8aadmEqAVFS9yfmzHhUU8UzvyNB4tpBG4NMLy2YeZWFN8 +QeNPA4QdMkT9VIvOHSUhQ3oORulXJl7a5tvbq2tyq3U4FIQBvBAO310bMBsQcKw+4ZIIS2kKSR5V +KZ9sPaWm1WqW37i2lSxH0gBBFUTLeamLW+Tg97cBt9StSEcAB/N9aAws4LgLCbTEbrSi3aWLo61d +1Kk7ifQ7+6vXeke2uLdCsvYLf46tThSk2zKuzBH85wjSBVeTeu3rCMPNom/tnFauyUdKfzvETyqb +fubpFs2wthtCUCENsSAPACKCk50xPMvt72L4nkt4Xa2SygtuIUkIIjeFHkfKh5huMWdk+W73Bb2y +UDu6WSpER4iYq45rwS9duV3Nz7Ssap+ceJ5cAJquM9o2QEIJ7wkKPCgnsHxCzxa1F0w6haSkplJ5 +UjdH5tdwe4gCRJj/ABzpo0lq3X2rbKGVLBBKe7J9OdUnpQzULWx+R7R3/KHk6VkH6CefvNBMdEmc +13PSMm1u3W12C3VBgFIlJ8Z84+uuvLIzaoKtyRXzpy7fXFjcpWypTa5ACk7Eedd99HmJt4zlHDbx +D6XlKYSFrTwKgN/roJ1xHdFY2J3mt3EmIrUCO7FAqhAVsRtThIAiCD5UigGCZO1bpSqNiaBR3SpH +CCKFvWNI/Fne7TLjY/vUTl6tJoXdY0KPRnfGP41sf3qCb6lP+h7h0gDW8J/PTWVp1Ilzk59E8HXf +tRWUFX6/5P4NWoKhHyizt/8AS5SnV+bDnRpgaU7xb8PzjSH/AOQNUZfsk+OINH/dOUr1eFhHRlgg +3nsJHl3lUBONvGxTBitUthPGZpVLx3kztz51p2xJ2igTuWu6ajLhvjvUs8XltEttqWI3IrZGFJWl +K7l4hSk6uz578KCtLQrXEz4RUknAHT2a1uaFkayI29KnWcIsrZ/tN1rSJgqmo1u5vLpt1p1pTUPE +srk7jjvQBvH04hhuarxzELlaGn0Q2hHBJ3gTXOefrwt5oadUl1p9h4HQvYxqkGuoOmF+1U0bJ25Z +aeWsKMqEq8QJrm7F8PcxfPTLKy12LSzD6mtaFJBkBQHHwoDBaYld4c004SpbRAKVDiAeVWFrPNi1 +aAugrlOgEHcbc6jsv263sBZtbhse0sICXE6eGw5elRSsJtA4/rag8Y+6gaZqzheXlyvsSWrdHdSh +KREcKgm8bt0IlxYTGxkQaZZvtW14u0D80yGwVRtJBqCxRy2ClOaU7bJkzvQSmI5mff2YgNjYeJNC +/MTy38auHnFd7VzNWW9uDDVtbgLed2QmOE8SagMesfZr0NKJU5pBVvxJoNMLt3HvyoI70zw3/wC9 +dP8AQRnVzD7K3wm5uGlWSTpSeaT4ek0CMm4GL68asw65LpCNCB4kcffRz6JMAbX0iXOEBxsJsWUO +IhI0qBH+PfQdDsrQ4RpcSraYml9AUkq2EVDW2GO2OKe0hsr1JIUUq2I9KmEPs6ezSpJWRtNAow2C +jma2TqA7vCvGFOCQpPpW5PmRHGg8UCUgn30LusgI6Mr8CZ7Rs/3hRWOyBznwoXdZRMdGV/wMuNfr +UDvqQz+CNwTw7Z0Ae9FZXvUiH70LiOAedjb/AGdZQC/riY9cZgybaXdx2QWnEGUKDaFIAUGV6tlC +eYq1dX1BPRrgRHO3/wCY1WuubhNnhOUrC1sX3LhtF61LjiYUo9k5JMVaur4kDoxwLf8Ak5P95VAT +A0rmB8a2w+1L9ylomATvHhSrTetMJBJPKp7B8P7BsFMdqrdRPKg8xK0Qzh6bdlYbkiSBvE71AnCc +TxHGRcqAtrdteyie84ANtuQq6KZSmCoaz4mtVpII22oIe2wq2t2tMOOL3lajJ3qDzc37Jhb2lSkF +YgL/AJm3GrhpUkn3VSOljMWDZawBy5xd4gOgpbQlGpSj5Cg59zRaP67w35beeZAUwp8gl1JO59QK +p/RVgS2XL/GL5am+3LyWg2jVI4QBB47/AAqbxvNb2a1vMYQ+i11KSEQ0QVeEq4/DzqwdGjLlvlkL +unHEPquH9bkAqRGqVAcyDvHlQXHEMETbu219aJKA+yhtxB5EDYn7KrGYLY21yEOoLayCFBSYNF+8 +w4sXFu+EqubNTRVp4qVpbBA8NzJk1TukZeMu4fdN4ZgaMTcaLIDIZDim9X0zMgwPf6UHP2e03Tt/ +bN2rS3HFpACW06lKPkBxppYdGmasQR7RiCRhtvpK4cV84QP6PL311ecm2jFs0xgybSxu1sIcDxZk +lJ4jYgmD9tVvFsoWiQ45mDHVvW8QvfsEQDChtufpJI35Gg57Tl/C7ZDjdtcsWz7SE9s65K3EhXPh +z4eFVvNtrg15b9ph63W32nEMtoKCS8Oa1K4DyFdE57s8k4060zqQbi2SG0sNgw547jYgcffQr6RL +Sywu6ZFtbN2zCXO0LaR/CECdz/jlQVnLWH3RxRoWoLEaVLUtUwf5225H30XeifGlYJnO0tn0ds3f +/MG6UgJgkyD6HhVUyW3b3l+wEpSxcusjSDJC522+HD0qy4Thq8VzrY2QcKmRqaeCO4oECAoBQmAR +QdTWTYCAVLCieFe3WFWV4pKnWh2iFSlSTBFQfR85eHD3MOxLvXlkoNqWf4xP5KvfVwZbIIOxmgi1 +4elgSkrIPDypqtlQE1ZnW0qbg7yKjLxlTKVAJGmOJoItepKR3gDFC/rJKJ6Mr2AdJcb3P9aig4VE +bDj4UMOsekp6ML2SILrUfpUEj1JmyjJrytiFOOnj5oH7KyvepPIyfcTzccP95IrKAb9dN9p7J1gb +a+urtkXjPefIKpDTgIMbE93iKtHV1LKujjL5fC+z9mIMeSlVS+tZh2L2HRrZt47Zi1vTiTQKO11w +OzcgzJnh4mip1YcNb/FTgDrqQrVbE7jh31UBOsLcdhrbZ0JiEDnUvbM9miIk86SsdK3VqaVqbTtE +bTzp4J5UGq0kxArRxsngDTkkgbcBWizwmRQNXGjvFBHrLZYxXHUYabJhbzDetC0gx3jEb8uHOjq4 +ElMzNUHpcfScEFmq5RbofVC1qXpgDfY+M0HI/YYtl1a3bmyt7C3LnZ26zHarIMaoEmOfhRKyM8nE +7Fm4tihLaFKWHNMBKtCiVkb89yKHXSo7a2+IlouqKWkKTDjsjURsB5wZnzq+9ByG2cCtmlKI1rB4 +mSSOBI5GfhQG51SMLwhd9d3wWtNqhaU6YTAG+mPEmn2AN2tvZm9cCW37sh1ZPiYAHwgU6Yw9rEcv +27ToAOgBJT4D9hpZGHNuhDK4T2YiOQoEsVtnlLtvZ7QOuIUdLmqNCSNxHP0oZ490dXl5jryPlJ9F +m6kOOslZWFKPE7kxwG1EZnGGnMSvLNKoatEEvOzEE8BNKNOYa7iSk2zrbtwWwpxSSVGOUnnQU+wy +Rg+HIQpNqhSkc1JGxoE9OeErv82Iw9huO9Ko2CUwkn7a6rv7VTiDpMeBP21zn0j26lZuvLy3e1Ia +CVSonhI24eCaCjWtk0xcW5xC8RbtWoCmm0nvEJMgGOG9WHo4dxRzOthdXGHutoQ+oF7V3QF8IJ3P +ED31EvM/InbEqZccvHNRURqCWwIjf3mpfJ79xdZywtCriXC+lxSNR7yAdyYMAbbCg6WbZXa4na4m +2SEqhm4HLSfon3GPjVub2G1QVtbC6wxbau6FoInwqTwl0uWqAsy433HPUUEnJg7CKbvoC0qB4U4k +cfqpFwlSjHKgg7pstrIKNPhAoSdZhSm+jW4kDvXLSfrP3Uar5jtGpmKB3WkXp6O3GuZu2p+ugn+p +aQrJTp0gaXHUkxx7yT+2sr3qWQMjL8S69z/pJrKCqdfhR/B21bIgG8ZIMcfm3Pvq79WMA9DeXdWw +9nV8A4qqf19Qn8F7NYBKxetD3FDn3Vaeq68X+iDA0L/ikKSNuI1E/toDCNKW+7tXjbknTMKB3pNR +BRusSRwpG1bQlZdTqle5M0D8pIRIMzWi0k7kVs26VARuOHpWy9kydtqBvyAoR9PDdy6W0NBOlLJV +KuAMnf3UXlQTx40JOmrHLe1vfYLllLluGPnSeRPn8KAA4sxhGY0IwXEbdAcaKSxfJRpVpJAjVzTx +MHwFGPLGXLXA+yw6zh1lgobSswZKYG9B3GUWzCEXdmXHGFPoShKR/Bq8FetHnJTbjlip1/6Yuz9L +YjvJP7aC9ZYYPyUyoOiQkApSQUp8hG23Davc2JxC3wi7fwlkvXvZHs0AgFR8idp8JpbKaAjCGwmS +EqKQeaoP0jtxPH31LPJ7VlSRsY29aClZTwFdnkq3axBhab19IuL8FYUpxziQo8Dvy4VL4Szh7XaN +WduGlIjVCCBvy4fZUi2T2imjsJmDSVuy6h59bjoUCrupH5I8KBtjTybTCLl8qCSlohJP84jb6656 +zk0FX+IS0pbhW2lKUnc7EzRw6QHpwL2clZ7d1KShIOpQG8SOA2Ek+nOg3iuFrxBV+Ge0S4HHXAAd ++4Dtz5GgodlhTuLOXV9blt+3t1qTcIKtK2FA8DOxnl76neizALh3NTd4ppRJCVIIJ+jJ+HCvcq4h +8nZdRamwLpecU4tLg1BaCd9W3Ab8TPCp/oVxC3s80Iw95d2lVwSlhLoEJgcoHODQdA2Dei1QmDJG +9LJZ7N4vN7H8pP8AOrGNSedOAQU7+NB6l1KyIO8SKb3L4QstEwTBn1P214NFvreIJK+fgPCoyzWi +/eXfPhaQh5SG0qVACdoMefH30Ek+4lCi2dwQTx50Detesfi6J4TeND6lUYbgqf7VsK0KSe799BDr +VOLPR8pCjwvWgY9FUFr6k8fgSuP/AFHj/eR91ZXvUm/0GV/Xe/XTWUFc695P4NtJO6ResHj/AKty +rb1YRo6IMvAAd5lZ9/aKqqdexKjgCIE/5Vbn+45Vs6ti9PRLl0Dj7Or/AIi6AqO6QtC1qjkBHE1o +HCEKCdlQSJ4DwmtL3UGwtKgkpM1s22OzCn1pJMcOFBtgq1rtRJKlT3jBAJ8p4ipIokAz7qq+FYnj +D+OutrtbVOFto2eQ4dQVJATEQdoPvqypdA3kEcaDCxJ5jehL0k4JbX+aF2D4Kn7tslsaZ2jaD4yf +qovJeBA9aGGbcYUxmS4XcLW2GwopMDToHL12nagDeMWrOQ8Ut38Te+bfeShTDYSrWUwZUOW870UM +MxBD2EX17bgFKrp4twdjCEkfZQn6VMHxDMuZrB/CXbd3tnA0u2dX88gkzqG26efjtRiwWy7DLhY0 +gr9ofQTp3nSdx+jQXTLLxUxdJVGpL6jtMQrcRPkR75qQQ+kKUkq3mq1ktSoeSlBQkttqEj6RKASf +MST7wamXwWX+2EQdlDwoFApS7wmBoB2ptbG6TdvOOvsG3OzTSEFOnfz4+tN7fEW3seXZNmS0NawP +yR51o9d3Vzj6mBZhq3aSfnCoFThkbwOAoIHON2HsXtbALWgBQVISd1KJgTwiEmR6VV8rgqexG9AB +0WjzgJG0mpa5Wm4zViN8hQcRaoXJ0gAdmgAAEHeFFcnjO3KkujlTSbK+uX/4MMLCpH5Ox4e80Alv +7O89rtLK0tlXK1EgISqEK27yp2HAbDyqU6MsLXhucEqWyHHGnNTbhSJIVPCCY9Kl8CtsYvbxzFuz +bu7dLmi3t1PFGpO41pjhyEceNPMpl+zzkwzeKLr2IK+ilOlCAlJIgevPnQGFNwrsCofSArXA7x29 +Sokd1KiJ9K2QzpSNUwRxpLB22MPs3xqVHaqWeZMkmBQa4u+3fONWdqVrLdylL+gwWiBqTPiOHuNM +ct3D14L9zswi1RcuIb1cVadp9Nqe4241h6HMQbMLQytao4KhOxPjwpnktl1rKNqHge2cRrcP9JW5 ++2gmrRsItgTBJ8KA3WzRoyUFJ4KvG5+CqP6GyGUgAcKBnWzQT0epIH8ub+xVBYepbIyQ2AQR8/Pl +84Kyk+pPq/A10HcBbo9O8jasoILr1ScBRBgh63j4O1YerOo/ilwHURs0sbf7VdV/r0j9wkqn8u3+ +1yrB1Z5X0TYFwjslgf2q6Aq3X+bSPCeNIYchNywQ5JTBETtTi4SVWy4HLamuEhSElJM78qB1CGG2 +7VpCUpnvnkkePrUQ/cXjOYmbFAUbd4FSf6McR9dSmINLUJCCptB1KTzX4D0mqhmPH3MvZgZvbxtR +sy+1bKITMLcCt/iAKAjhsJQIiYoW9Ilp+6V44yg3DhQIQ2qTJ2I08440TWHw/bJdSkiUzvQ2xxdy +xib7rDalvF8nYctXCaAMYfjWJp6Q8Js8SY1D2xBCwkoIIOwE8eKZo34OV+zW0pK+1vS4QFcAttdD +XGMYwu66SLG4xKzadvWLhDbLikFKgeB9QJPwotYez2YwxI4FDK9/zx+2gTynCb20WVhRfsQAmBKA +hRmY33Kvqqw4mlCLRxxwEpSJgbk1XsHhl7D5b0Bu5fZU6Y4EylBnfcmdvDzq4ONpXG21AN8i4ZmH +D845gfxLsBZ3LiV2znaa3FbniD9EAbRw22qw2bTFniV865dXLj4BUpK1Hs+E90UviDqm8UJgCFAH +zBH3iorPVwm1wdy5ceLZdSGUKClJAUo7GR/jhQVxntG8v4ze3LTSHnWtB0DYFaiYnme9x58aQwNt +bWT8XdSrQPZwkq8CZn6op1i/zOT20JVPbuBSTxkAEj7BS+VOweyhiDlwvTbuFxKlEcEgQf20AJuF +Y1eW9vb2jtxpYK3FqZdG0ERPDxiiV0WYWu8xNGP3ilpdb7gSsGCqIkVBXgwDCMFQ40t5Srx5QZd1 +hSVRAkDkB4bVa+iS+YcuHMMR2kBHtACySdzvx5cKAj3t8pi1WUtqWoJkAVC/L9naWKbh1Sbi8WUM +9i2QopWQYkDcDjvUvi94zZYY/dOphttsqJAnYChDlN22ds14xiS/ZXrm+cuWme00rUClPZpUBvJA +n86gvrz1zieBui4T2NzeOIY7IHZvUdwPQA1dLVkW7DbKYAQkCqlhlstzMOCWOgDskru3gTMHTpAJ +9VH4VeFp+cMAbeNBouNEBRG24oJdbFH/AIcpM8b5refJVG90d0bbmgp1sEk9Hze8RfN7fmqoJTqU +6fwKfE97tndp80VlJdSrbLFyJnvu/aisoIrrzpP4OoVy7S3+1ypjqvuT0SYIkk7B0f71dRnXlSDl +hKo4Lt/f3nKfdV0a+inByPyQ6D69qugM6BLShHEUyw7a4dQDMHantuklBHKmjbfY3ijBAO9BviT7 +bPdeMJcQQY/Z9dQV6S4n2e+CXgp5tbIUmSUpgyfMGneeGFuYCt5DikKZUlyQJ2Bk+u01st1tWYbS +zCNZuGlKkD+DSIG/hMigstuEhlO0gjaqRmppJvXvY3GW7pSiYdMJkDjPjV8Q2EtgDkIqi52tmlG4 +L3dE90zxJHDyoAnjDdwzm21uMX7uKXVwA2yIKUJTxWFDZRO3186NzjqWbXCVE8EIT8Ck/fQlunm3 +834azcWVtqt3VJQtLkqEjf7d/Si9i7KU4HaPJ4MupmPAgj7qBtdtqRbX51hPs16h8o2GuTATJ9QR +5gVamHSS2VRCk+FRl9bj2m7HY9qLi2JCAY1KA2G+wM86c2F12+CW7y161pQNStpkcZjnQNcetwhZ +dA7y1JHHjVTz5cLdcw2wYQHpd1OTuECISvbeZkDx34xVhz1iAscKNyYKUd4jVHI0PsOujfZgaxC5 +uVIbZSFFtPNOkOBRj1IA8vSgks4OMpNtZtAFthWohJHArA4egVTjCbFf4txag6XHWCtfA7rOoj6z +UFfF66XcXUavablQSTP0Q2qOJ/pD3ir4y2G8Hbti1rBbCVDgIAoOfsaxFOG481hC3LZ21QrtWm1t +hayrgrSeRIMRwgUR+ibCWkdpj3cC32g2hCTPZpHI+e31VRs2ZYtHLxePKJUwpxTSWUcdYPDUOCSI +38iKu/RG4ub+xDehhhSVISeI1DfhtxFARnG27izW28gKQRBBHEUEsFwkXnTJcYeo6rSwWl9CI4bC +J8d/so3gAtQKpOG4a3adIuL4i0nvOWaJ24qkxQWzKzYdxbEsVIOnULdonwTx+s/VU8rvKK5nwApD +CrX2TDmrfYECVkCJJ3P106aSJ8qBEqIAEDhQZ61xB6PmduF83+qujY62nwoJ9a8j8XzQif8ALkfq +roH3UsP72LoT/GO7e9FZWvUtIGWn0wZK3if0kVlAy68IJyxO8Tb/AK6/vpfqquBPRfhQJjvuiDz+ +cVSXXg3yuE+HYH++uvOqopA6M7DWB3XHY/TNAemBKCBSdy2lDqSY350lZ3KdxIn7a0xJ0ltCkmCn +eg0x1CXMJfbIiUECee1UToGzLe461jFtibRVcWF4WUPkfwiOQnxEVaMRxGbRYWZEGo7oktLS0yym +5YA1XTzjyz4kqNBeytU+NVvMFlbYip5t5TqVEBIUhUaTyPhU4t4p4c6qGbHL9DinLI/OIlQSTHLj +QDfOeCsYNmBDlpZEvKR2i7kypSjqg78Ad55UUFLL2WlIUkmWwRtPDehPjeM3t04W7nuBtxsFA2JK +l/ZANF7C1JVh7bSu8kpjc0Gl1iLbVrh12tQRKkoJJ2hXdH1kVtgbgFvdWyne1LbihqkHUPHbn4+d +QePYbcYng7Fqw5p7J4FSvNCpA+IFLNXTeFIxS+dSUKTal5QJ2UUpMke/9njQD7NmNX+dM7KyvhyU +2+HWw7R+4c37VKFaVCBwEggTE0/dbfQWbewabsy6pWs9nJDadjII2Jnlt4VZzY2eH4cvFbawtkYl +fNN+0PBAG8TJ99Nra0Q7jTSyhLa3GktrK3ZEDcpTzJkn4cqDW8swi5wu23AbaU85pHFS1AD6pq2L +HzACRAAqKdQh7FiscilA9E//AO1KuL1IMcAKATZrFthgdwnEb1NjhhJdU+133DqV3U6dJj135VNZ +CxrK7PZ4Xg945erMBb3YmSTJ75gR68KpvSFdtXV3ibz60e0W1ou5LSRsEAqS2k+oMnwIqMyLauYM +3heI3lk9bKfulWt20pWkgLIgn3EEUHQEd07kCq1lwPO5qxRTgGhCkBJ8QBP2mpe2eXZdnaP6ltaQ +lp8qmT/NV4Hz5+tQVveLtMcu9AkuOoB9KAgNqUobEA1oHFa9JFJ27mod0+tK7SFzvzoN1lWmZ4UE ++taf3gsTzvkAforo0uujSQN6B/WqcKskWyRzvk/qroJfqYT+DTuw2U9+sisrfqZD97D3kp2fXUis +oGHXc3y3Eb6WD/vF1TOr7mvCMJyFbWt3i1nbPJdc1IceCSJVI2NXTrrtzgIUebbH1OL++uNnEkHa +RvQd42PSFlkAFWP4bI//AJKPvp09n/La298cw0+l0j764C7yTxPrXhKualUHcGN51wJVo6pvGcOJ +0HhcI++k+jDOGCYZlJi1uscw8LC1qANygEBSiY4+dcSHUocTvXg17gK4UH0Jts/ZXUolzMOGJA8b +pG/11Vcx59y6rGJbx2xcbKIMXSAkHj41xBqXEaj8a1UTO5NB1FjucsuqddGH3tj3HWe0V24AUe/M +c1RIPGN6I+HdImVm7VtCswYYCEj+Up++uFIJ8ayFfzj4bUHdFp0j5TQ++hWYcNSgr1g+0p58ai84 +dIeUVWi1s47h90lbKmXbcXAhaVbevjw8fSuLAFR9I153juSZoO2nek3J5Nuk5isAlpIIT2oIGxHv +2Net9KOUWFBacdwwlcqWQ6JnhwiuJAFExJisg8N6Dta16UMn9sFuZhsATqO7nifup4elTJn5OZLD ++1rhzSrzrIJTtsaDoPpMz1gb+NG7sL21uBd4Y7au6FA7laon0mansdz9lrEsh2S14tZ+2pQytbYW +NWtIAPv2rl7SRsTNbBJ8aDtW36UMn3Fg2HcespU2nUC5wMVBPdIWWk4s1+7to4nWklwrHAHnXJAC +toJrfccKDuWz6UsohO+YLEf/AGU5b6UMnkE/hDYE/wC0rhi1WG7hC3UFxCFAlMxqHhNSPttq46Sq +1S2kuqXAPAEyEjyHCg7Vd6T8omNOYLDf/WULOsDmzBMeyqzbYdidtdOi7SvQ2qSBpUJ+uufFLK3V +rQAlKjIA5DwpdqeZJoOu+pjtlh8Rtqd/WTWUr1N2+zyo54rLiuP9NI/ZWUDLrop1YAgaTu21B5fw +iv8AtXHrzMmNJ25iuzuuIytzLgUB3UtN/Hta5DW1J+jQRBZAMwTWBid441KlqTFZ2IkkpE0EZ7OJ +3FeKY7xhMVK9l4CvOy24cPKgiCx5D41qWoMQRUuWSREDjNeFkapIoEEi0Fro1Q4EJAIRzBnf69/S +lXrlpSrwtdnp1ksgtjgVSeXhtXpYT/NE16GeGw28qDxm4bLlmXez0hep8dn4K25eG1aBVsm00KIL +obUmQjiZOx90QeVO2WmFFKFMjUTx1QPftTj2FggkBgDURBe3+ygi7z2Zxp/sVJSlTiS03o3QnfaY +9PXjSeG+zNsuJfSky82RIJOkTq5c5G3OpP2JorCAlkd0GS5sd/t+6tnbW3aX2iW0KSFboS7Mj4UE +Y8m2DKezUndrTu3BCtczw8K2U1a+03L6XG9Dpc7NGg93+by293hSzrLanCW29KTwSTMV4liOVBHP +WiUL0ocS5wkpBiffSaWNztvUr2G86a9DMDYR40EX2BE6RNbJYPGNvGpMMA8q9LJ8B8KBghnfhSoa +QdwmDTzsiTukVuGjEQAaBu22EwIpw0gTsK3S0rbu07tmVTPCg606nrShk5bihA1OJHn3xWVKdUtC +R0bJUEgEPLB24nWr9kfCsoNOtVaOXeUEttNqWsplISkknStBP1Sa5NXhS9KdVncSAdXcO/hX0PxG +wssRt/Z7+0ZuWpnS6gKE+O9RRyblgmfka2HpI/bQcDDDGAynVZ3Xac1Rtz/7Ui9hqVbtMOpE7yJr +6ADKGWxwwlj4q++tGcmZYZWtbWEMoKzKgFqgnxiYoPn/APJrv/oufCvBhywf4Je3ka+hP4L4BEfJ +jUep++vBlfAAZGGNT6q++g+e3yavj2S/gaw4a4Y+Zc/RNfQo5XwAmThjU+p++vPwWy/M/JjXxV99 +B89Pkp4meycj+qa2GEvT/Aun8019ChlfAASRhrUnzV99bHLeBnjhrJ+P30Hz2ThL+/zDv6Ne/JFw +Zlh39E19Ck5dwVPDDmR8aw5dwX/49r6/voPnl8kvD+Idn+qa1+S3Z/gnI80mvoactYESScOaJPmf +vrwZZwEGRhjIPv8AvoPnmMLdH8W5+ia9+TF7y0ufSvoYct4EeOGMH1BNe/g7gn/xzP1/fQfPA4cs +H6Cx5RXhw9QH0VV9DXcsYAv6WGMn4/fSasqZdJk4Wz8VffQfPgWPPQfSsNlvOk719Al5Qy2RvhLP +xV99aLydlk8cIY+KvvoOChZ4fKtSbgbbcONYLO030h7yMV3j+BuWNR/ce34+f316jKGWwSBhLIB2 +O6vvoOE7e0swiXRcSBtoAiYP7Yq69FHRxdZ9xt2zs1qtLW3QHLi5WmQkEwEj+keXoTXXyMpZcSNK +cJYA4wCfvp5Z4DhNqoqtrQNEiDoWoftoGeRMrYdlDBk4PhfaezoMhThBUSSSSSAPGsqfSAlISOAr +KD//2Q== + + + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: image/jpeg +Content-Disposition: inline; filename="custer.jpg" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/4Q1kRXhpZgAASUkqAAgAAAAIABIBCQABAAAAAQAAABoBCQAB +AAAAyAAAABsBBQABAAAAbgAAACgBAwABAAAAAgAAADEBAgAOAAAAdgAAADIBAgAUAAAAhAAAABMC +CQABAAAAAQAAAGmHBAABAAAAmAAAAOYAAADIAAAAAQAAAGd0aHVtYiAyLjExLjMAMjAwNTowMTox +MCAwMDo1NzowMwAGAACQBwAEAAAAMDIyMQGRBwAEAAAAAQIDAACgBwAEAAAAMDEwMAGgAwABAAAA +//8AAAKgCQABAAAAyAAAAAOgCQABAAAA9gAAAAAAAAAGAAMBAwABAAAABgAAABoBCQABAAAASAAA +ABsBCQABAAAASAAAACgBCQABAAAAAgAAAAECBAABAAAANAEAAAICBAABAAAAJwwAAAAAAAD/2P/g +ABBKRklGAAEBAAABAAEAAP/bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAk +LicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIy +MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/AABEIAIAAaAMBIgACEQED +EQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0B +AgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpD +REVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmq +srO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEB +AQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFR +B2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVW +V1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrC +w8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEAAhEDEQA/ANNRzUiIfSlR +DmrUUfrXOajEiz2qZYPap0jqlrOr2+i2RkkZDMR+7jLYJPqfanYCd4OOlQGe2ibbLcQo/ozgGvPL +vxRqE6kSXsj+yfu1/JeT+dYsl55m7JUZ6kD+vWiwrnsAlgfG2eI/RxUhjrxlLhgMLKcdua0YPE2q +2mDHfSMB2c7h+tFgueqeVimmPiuO034hKWEepW4APHmw9vqK7G3ube9tlntZUliboymiwXECCgpU +oSnGPNIZEqgUVKIz3opAVUXDVajFV4/vVT8R6yNC0SW6THnsfLhB/vHv+AyaaApeKPF0WjK1nZ7Z +L0jk9RF9ff2rzC71Ge7naaeZpZWOSzHNVZriSaRndyzsdzMTkk1GFLHABP0qhEhnc5xWjpmg32q5 +8oY9M96hs7DzHG4MBjJJGK9j8HW1ta6ckqqHVunHQ0BY8wl8H6hDGWOcjttNYtxbzWxIcHivoXWN +X0nS7YzXh5I+WMDJNeY+KTb3umHUotOlhgkfYjkfqaLhY4ESZrb8Oa/caLqCMrFrZyBLH2I9fqKx +Y4mkcIilmJwABkk16R4V8BmDy9Q1dfnGGjtj292/woCx2qjIB9aftp+MUvFSMYFopxopDKCDDdK5 +L4i2V9dWdlJb28kkEJcylBnaTjBP68116ctWjanFUhHzpit7SLa0t1W7u5lCk4AxmvZbzwvoGoOZ +rnS7dpDyXUbCfrjFeZ3WkW0OuXEcMMbQK52RljtA9jTESWNvDqExntAJDu2YEf3R6/413Xhq7itb +IW88MgKMct2zmvN47h9C1WCS1uzGGbbIqnPynsexFelLMloUMjF4pVBEh75oAvX50q7+RIXknkwG +Pt6c9q0bu00e/wBEk0m/kht4pE2KrOAwPYjPcVmWlza2TSXTbWwMgep9K5zWdavNSmISzS4bOMjA +x6AUhm3pHhfRNFVZLG3EkmOLiQ72P0PQfhWnITUVjJNLYQPOmyUoN6+hxUjZzQAwr7UgWpKCMUgG +7RRRzmikxlGNavwDiqiDB6VegGQOKpCJygkhaM9GBBrzHxFp50+5la8tnO4DbIg6LnqP0r0S/v5L +K1MtvaS3bZKbYcHDDqD7+3WvONT1u+1W/jbUSVtY3yYEGAB396YjCtLQXt1mOBhAjZDPyT7V3GkX +bJbtp92pe1I+V8ZMf19qzta8T6dounx6dplhbySkbi/U4PIJP9K4y88Q6reYMk2yPOQiKFU/h3oA +73VLC6trFrmCVJ7dTn5X6fhWLo3ih31i2N3Bi1iYqWjHTPc+uKg0zxCb+2NrNaHj75hOFI9wKs6j +FF9hf7CotmjAZWTj659aAPUQVdA6EMpGQR0IpjCuJ8GeKVNtJYXshd4xuiKLyw7jH9K7SK4guGdI +nBePG9Dwy5GRkHkUDFA5HWnMBikIwaKkBo64FFAPNFIZVQVi+IvEk+mSJa2OVnxueQKDt9AM1uov +NedeIJkufENyySEAEIMHjgY/mKpCZaOri/hjg1G3jnhRtylB5LofVWXHP1zWpqF1pt1pg8+4jmlS +PbA6o32lj/02ydpGOMjr+lcsXliHOJF7jHNQSyAYmiPyA5I9DTEQnTVimMwywHOD6VoafJHp13Hc +pZW92gJIhnGUORiljcSJnqDUMTBQ0fdf5UAaFhANDj1STUbP7PJN+9EQHHlnJUKPTJNSW4G4jqCB +1qKKzu0EeoXV0k0N0CqiSXc4C5HI9OaSNsXCIvCg4oAzNbj1O01iDVXhS3km+dPKUKpHTOB61b0j +xNLYayl3JGBE6COWNOAQOhFMv7QGCa7N5E489kFuZCXTHOcdhWMGBJJ6CgD3BJEliSVTlXAZT7Gn +AA1xvgnXJrxX0+5fd5SAxEjnb0wf0rshjFSMTABopwxRSGVgDn615TNCTNMrctHIwP516xzuGK8z +1FBa+I7yJgNrynH48/1qkJleOZo0GMstMlDy5MKxsx6g8GpGUxSnb9084p5RZUyuFb06UxGfazPE +WicFWB4B9Kk3fv8Ad68VDOri4BaNt443D+tEchKsfegDQsVQy3DjJcR/d7dR+Va9lDGLR/NkLOzE +gAenTn6iudsJtmoMeoaMg+3IrbtpJVjXa+O/NAGK4Q3M5bG7ewz+NZqAZcdlY5q3bxXOrasba0QN +LNIxUFgo7nqa1PCPhttZ1a6S7Vha2rnz9p5Yj+EGgDLsZ7i0nS6t5WikU5BHp6V61o2pxatp8c6M +vmYxIgP3W9KI/Bukyu0stmgZkwkSsVWMepx1NcOss/h3Wp47K4WRUfa+4cSD6UmM9IGM0VQ0zVrf +VLbzImxIB88Z6r/9aipAn3ZNcB41tGj1gzKMGVFcH3HH9K7pDzXJeNCrTRuzHem1AueMNuOfzX9K +pAznLa7juUCSEJMvr3qZwydRx6isi4ty3zJw1TWksjrs3kOOoJpiL0twwXlCwHoKxri6WFj8pG/k +A1fe6vIODtKmu1+Gc9vcazNHeRxyZh+RXjVuc8nJ5oA4zwxp91rmpSW1nGGnZMZbgAZ5JP5V6Cvw +213Jb7TYgbcKPMbr/wB816pGsESARpGi+iqAKT7VCPlMi5+tAHisPwc8SLJk39hH/tpI+R/47XWe +FvBeqeGtHnhlENxO8hlYRSH5vQDIHP1xXoSzow4ZT9DSvIO1AHiepeMNUWaWCBPsgDFWDrl8jsc9 +PpXMajqdxe3BuLt90zAAsABkDp0rq/ixpTWeuwanbDC3kZDr/trwT+IIrziS6uUcJcqGWgDpPDOo +pb+ILctJxITGQffp+uKKyNFtjc67YxqdyNMrfgDk/wAqKljR64vB5rK8Q6DaapCtxI0qXC7I0Kth +eXAGR3+8fzrTDYNQavKRol465DpEZFI6gryP1FJMbPMEcqxRhytNkwrLInDDriqST/ay3z4lB3A+ +tSrIWQq3DjqDVEl9h9oiDIC2e1V4H1KxvEuIGe2KHPmZxtqkbuazYtC5FVJ767vG/eyEr6dKAPQr +P4k6tZRHcy3KKeGlXJI98VuQ+OdM1EB3nubN3IASWM4Y+gIzXksN4YcB49yjsank1L7Q4ZvlVfuj +3oC57To2vQX8jJBdJujO0hnCnP0ODXWxC6dBuLL6ZFeP2/izwp4hsIodejmtNREflvdRICjnszY5 +OepGOtcvc6vqeh3Ij0jXbnyifl8mZth+maLDPU/i9Cf+EOt53YB4rtcHvypBx+leILdgDbICwrqL +7TvHPiVo7fUTdTxp8y+dIAg9+uM/rXS6D8PbDTQs+pst5cjkJ/yzU/T+L8fyp3AyfAmiXHn/ANrz +I0duqkQhurk8Z+mM0V6BI+FCqAFHAA6CiobA/9kA/+EMRWh0dHA6Ly9ucy5hZG9iZS5jb20veGFw +LzEuMC8APD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQi +Pz4KPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUg +NC40LjAtRXhpdjIiPgogPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5 +LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgog +ICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICAgeG1sbnM6dGlm +Zj0iaHR0cDovL25zLmFkb2JlLmNvbS90aWZmLzEuMC8iCiAgICB4bWxuczpleGlmPSJodHRwOi8v +bnMuYWRvYmUuY29tL2V4aWYvMS4wLyIKICAgIHhtbG5zOnhtcE1NPSJodHRwOi8vbnMuYWRvYmUu +Y29tL3hhcC8xLjAvbW0vIgogICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50 +cy8xLjEvIgogICB4bXA6Q3JlYXRlRGF0ZT0iMjAwNS0wMS0xMFQwMDowNzoyNyswMTowMCIKICAg +eG1wOk1vZGlmeURhdGU9IjIwMDUtMDEtMTBUMDA6NTc6MDMrMDE6MDAiCiAgIHhtcDpNZXRhZGF0 +YURhdGU9IjIwMDUtMDEtMTBUMDA6NTc6MDMrMDE6MDAiCiAgIHhtcDpDcmVhdG9yVG9vbD0iQWRv +YmUgUGhvdG9zaG9wIENTIFdpbmRvd3MiCiAgIHRpZmY6T3JpZW50YXRpb249IjEiCiAgIHRpZmY6 +WFJlc29sdXRpb249IjIwMC8xIgogICB0aWZmOllSZXNvbHV0aW9uPSIyMDAvMSIKICAgdGlmZjpS +ZXNvbHV0aW9uVW5pdD0iMiIKICAgZXhpZjpDb2xvclNwYWNlPSI0Mjk0OTY3Mjk1IgogICBleGlm +OlBpeGVsWERpbWVuc2lvbj0iNzU1IgogICBleGlmOlBpeGVsWURpbWVuc2lvbj0iOTMwIgogICB4 +bXBNTTpEb2N1bWVudElEPSJhZG9iZTpkb2NpZDpwaG90b3Nob3A6Zjg2ZTcwZTQtNjI5OC0xMWQ5 +LTllM2YtZDQyZjM0NjM5ZGJiIgogICB4bXBNTTpJbnN0YW5jZUlEPSJ1dWlkOmY4NmU3MGU1LTYy +OTgtMTFkOS05ZTNmLWQ0MmYzNDYzOWRiYiIKICAgZGM6Zm9ybWF0PSJpbWFnZS9qcGVnIi8+CiA8 +L3JkZjpSREY+CjwveDp4bXBtZXRhPgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +IAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAog +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAg +ICAgICAgICAgICAgICAgICAgCjw/eHBhY2tldCBlbmQ9InciPz7/2wBDAAUDBAQEAwUEBAQFBQUG +BwwIBwcHBw8LCwkMEQ8SEhEPERETFhwXExQaFRERGCEYGh0dHx8fExciJCIeJBweHx7/2wBDAQUF +BQcGBw4ICA4eFBEUHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh7/wAARCAD2AMgDASIAAhEBAxEB/8QAHQAAAAcBAQEAAAAAAAAAAAAAAAIDBAUGBwEICf/E +AEIQAAIBAwIEBAQDBgUBCAMBAAECAwAEEQUhBhIxQQcTUWEicYGRFDKhI0JSscHRCBVicoLwFiQl +M5KissJDU9Lh/8QAGQEBAQEBAQEAAAAAAAAAAAAAAAECAwQF/8QAGhEBAQEBAQEBAAAAAAAAAAAA +AAERAjEhEv/aAAwDAQACEQMRAD8Am8Hl60dEGN+tAbnalEBIr5+PUIoGcmllXPSgkW9LxxntVBET +2pVY/Y0tFEe+KcpFRDYRDHSjrD/pp4sdHWIUwMHhwOlJcpXr0qVeEEd6aXCxxRtLK6xou7MxwB8z +TDSSpjelAtQFxxvwjaSmGXW7dnH/AOtWkH3UEUeLjjhGXATV0z7wyD/61cNTTDbG1AKcdKjBxXww +RznWbUA9Mkj+lSem3+n6lE0mnXtvdopwzQyBgp9DjpTDXGQ0mY6fFD6Vzy8DpTDTMR7UOT3p0U9q +KF3oaasm1F5PSnTr3pMAb1FJrGwGRRuX1o64xvRtqYG5X2oyp7UpgV1agCKAQSKFKZHLQozUEoNL +Rj3oBd+lLxJsKrTsaEmnccW+d65Ehz0pzGp9KuI5HHv0pdU7Yrsa+1LqmSBitSAsabdKrXiLxfb8 +IabG6xR3V/O4EVsXweXuxx2/maX4w410PhmJknlWe7A2gRhkf7j2/nWCaxxK9/qs+ohea6mbJuJd +2A7BR0UDoKuJaud54gcYX0Jk/wC46Lbno5T4sf8ALJ/Sqlq+sx3j/wDiOoahq7A5HmSFIwfYf2qA +uLxpn55ZGkf1Y5ptLPynAxVxEsdQXcwWNrCvuvM33phM8jS+Z5rZBz1pn+JxuWxRGvxzbDI9aYbE +rHfXgGFuX/T+1OrPXdZs3LWt/NCx6lTjP2xUGl3EO9HN3Gds/ah8XnTvEjiiywZLsXKj92UBs/ff +9at+heL9jNiPV7FoG7vCdvsf71izzAocGkWkyD1phuPVuh63o+uRc+mX8NwQMlAcOPmp3p8UA7V5 +Gs725sp0mtLiSGRDlWRiCD7EVq3Afi5KrR2HFA8yPoLxB8S/7x+8Pcb/ADqYNfkA3pLlpWKWC6t4 +7m1lSaGReZJEbKsD3BoKtYWEwvSjhQe1GA9aOoFA2K4JoAb9Kdcq5zQEagnYVKpALkHrQpyAB0Az +6UKioTlIpxCMikyMiloU2qwLRfMU6iGe4puin0p3Ah6kYHf2rUQsgVFLuyqoGSScACsj8TfFURPJ +pXDEo2+GW8Hf2T29/t61D+MHiM+pTy6BoMpFhGeWedT/AOeR2B/h/n8qyiSQjuc961ImndzdzTyN +NPK0sjHJLHNNZLk5601klY96SZqrOnRuipJApKWdn70iuTTqztWnkUAEg+gqp9JRrJIQEVmPsM0s +LG9dsC2lyenwmtO4D4PZ3jmKeYp64GQPnWoDw8WeBZIrZUbqMj+dTVx5l/yvUACfw0gx1yKQaOaM +kNGy/MV6UuuALhIyGRDj2qj8W8E3do3mCDIJ3wOgppjICzD2owmPepnWdKaAtlSCO2KgGBBwe1VM +OBJnYUA2BTdTR1zQaD4UceXHDd+thfyNJpE74dTv5JP76+3qO/zr0PHiQB42DKwyrA5BHrXjtc16 +l8Lo75OBNIF+GE34cYDdeTJ5P/bisdRqLGq980cLjpXQpB60dQc1lonymuBd8UsFocveoogXahSm +2+1CsqglI6E05twu1NU69KdW/XFaiHcYXOKznx44xOi6QvD2ny8t7fITOyneOE7Y+bbj5A+orRlK +Rq0kjBUQFmJ6ADqa8l8ba5LxDxTf6vITi4lJjH8KDZR9ABW+WUY03KMCkC5cnJorHJoA4rQ6elBI +2kOFGTjNGUcx32qZsdAubiESoC+eyjcUQwgspVTmZod+ilxk1a+EtC8/UIeZ+WOVSSmNw3/WKjG0 +60tpbeCZJTLIQVKHIO/Tb5Vb7Vbhbxr+1jES82Vjkckuo/d6bHHr7VBtXhrp8ttaxhreGKFfzcw/ +X1rTbdomQBQCpGxA61ROFW/EaBDcsQrcgyo3GcdKnLNTHmWV0VTjdTjb5f8A+UaieFjHK/mHBQZJ +J6Yqr8XXPD8UDie9tkC5BzKPsKrfGnFGu6jM2k8OlxynlaY/EgPYbd/sKyjTOFeK+JeJmtdUSRgk +gDM/wDrjI9fuaiadcY22k3EL3MbRgcpCuB136VjOoIou2ZNlJyK9NeKfhUR4fwS6aZDLYEtcIDu4 +I3b6YrzJdczTNzdtvtViEQoNKwxs7qiKWZjgADJJqS4b4e1fiC/Wy0ewmu5m68i/Co9WPQD3NejP +C7wtseEuTU9UaK+1gj4SBmO3/wBuerf6vtVXFO8K/CR08nXOKoimCHgsGG59DJ//AD9/Sti5eu1O +5zzHem/b3rPqyCgHNGUUXau5qK6a4a4DXcZqApOxwDQoxWhWK0gsdDinEGAelJMgxS0CgVqMmHHM +7wcDa3LHs4sJsEdvgIryU29ez0tobmNre4iSWGVSkiMMhlIwQazXjHwGtLsyXfCt8LWQ7i0uCSny +V+o+ufnXSMvPGN66BVl4m4H4p4clZdW0W7hQHaUJzxn5OuR+tV4Ieb8pG/pVE5w/odxcSiaWNhGm +CB6g9/lWn6XprWmnk5jifPQ9MHPX5AVQtK1trTTrlDkyOnlg+m2/9BTF77W7iN3Ms5UAFiW6elRF +6gsNQZ31aOS3gnjCrIzDnJU9cdh9KXKZto52hYWkcgAxj2OGx03qF4D1qOyv4oNSvVCznq+8cfpz +fP8AStj/AOxtla6VJDaXLXMDp580gwU5zvzAg7/agsXBN7pqWqWYYK4QOQTsexq4wjTrpArMvKw+ +1ZXw1oyxW8byOfPyc4bOKmJWvLRSPMbH5gcYxmoq5axacP2ulSAzx28e55om5Wyfl3qJ4Jt9Gtbt +ruCGeaRsBGlkJyM5O9VaOG8v7oJzFhnY9Rg1oGh6BIlp5WQVC9em+KKstmq3isvLEI5MgqNwfXtW +O6j4EcH2/Fd7qF811PBNKZorQNyRIDuVyNzv7jtU5xBxbxhoDfh7Hh6wEEPWV7gF3/2ov9a5oPE2 +rcUomoXUFvBaRgoAHLSM/cEY+ED077VUSem2NhpNmLLS7K3srdOkcKBR+nU+9CQk96OxBpN8YqVT +dzSecjpSjgE7UFUBTtvQJMoAz61xRntSpAoBRisqIF36Cjcu3SukUBt1oOcu9CjKAaFYq6r2AcZI +pxEvTem4SnMKYrUZPrUbipa2bG9RNqN6koelbglElBXBGR6Go/VLezNlPKmnW0kgQkc0S7n7UvET +iofjLVrjS9PDWyKZHzueigdT+o+9VHmziHSiuu3d6UjJmdZ1B/KBJvnHrnI9iDUqAlxZLplxp6lF +AdnXClhk7j164ovF/Pa6v5v4ZppJCWl5JCNic8uOg3Pak5+JntoRaS2DRyQrkCQbgEk9vcn70ZV/ +XtFktLCS4ihcRscDKbjpt/Or34IW8eraeLS4vrkJbyeYsAlbkLDcZXoaoGtahf3/ADT80nk8u4J+ +EbdKmPCrU7jh/VI71gTaSEeZj933+VFbglytnfGKdVUMxOMGpaSS3uYEBQZGQDmm2o2FvxBpy3tn +MAWXmSRNwfqKqY1HUdKumt7wFcbFlG30qYrQOHohBdBlwA24wOm+f+vrSfGfF8nOdN0+8MKQjMrq +uSx9B/aqmeJkS2llRsyKhKgnHNjNQcU97qzNGzRKZy3MScg/P5bUDbiHjp7GZFt9OlupWBBeQkkD +tnHqMnFO/Crij8fxBLbRwGOKaIyEZ3LZzzH7kU2nsI9Jljt5ktnuZRhZMZ39fUbZH2rQ+H+GtN0x +zeW9nDFdSKFkdR1wP0+lBNk5FEkyRSpGBik2HzqBHFGA964QAcZoZHTNWtDADHWuMpxkGuLgnFK4 +GOlZoR5cbmisTncUs4xtSRHM1AF+VCjqu2TQrNVAxg9KdQj1pJBvTmMHpWoyc2y5bYU+jUimtuu1 +PIx9a0HMewFR/E1g2oaXJChxJjCt6Zp+uwBpxFvjeqjzxfF9O1r8XeIwYebHKh/ccMcbf7Sp981A +Xer8P3Mt3eXrPJMIisUUSgZboC3etw8W4OFE0sXWs31tYXiDML8vNJJ/p5Ruw/lXn6/01dXuGvtO +tMO8uJQgPIpYZH3wftREUbt7pvwFkk0dtLgSK+Pi9/b9auOkaekdqEIxt27Va+A/CfWbuwTU4tPE +kL5BmdwoGOu27Y+QNPf8hcSyxxhAiNy87HkUn5tj+9Ax4M1274duxDcRmfT2OSq9Y/cf2q7cQaVb +a1apcWEsTLIAUYbA59ao2qXXDehoX1PU43cDaKE4P1JGfsPrTXhjiXUtUMknDn+W21ssn/kTO3O5 +9cbkfeinXEXDuraYhafSJJ0XOJIGz+lUK/4ku9PZ1tw6GPJIdcEe1bXJxWLfTHg1WExl15SVBIBP +p9axbibSdT4g1aX/ACyyldJGCmRl5VC+pJoENB41upOIrK/1W1W8tYHBeHJGR6/Mdd69P6Ve2Wp6 +bBfafKJLaZOZGH9fevMs3DtnoFsJNTW6ZCcGeJQVQ+46/fFX3wr4ls9Im/Cxail1pc7ZYdHgb+Ir +6euD70I2Vl+HNJkUvsyBlIKkZBBpJhRSLYwaKuD1FKkY61zAA6Vm0EQKW2pZeXuKSU4pQnbaoorh +TSZ5RR96IwqDqnOcChRosAdaFZqocYB6UrG2/Sk1BzS8Y3rcQ7tj7U9i3ppboaF1r3D+l+bFf6xY +296q/s4ZSTgnozKu+PatodalfWmm2Zu76dIYV/ebv7AdSfYVmPFnirO4ktOHoDD+7+JlALf8V7fX +7VfOEtW1K0sNSng4j0Xi7VrlwbeGS9NmkSY/IkfKwXt8+5rJ9S401nT9TOleLPAyzo7HlvIIBDOo +/iR1+GQD2OKuMqNftPqd1JPqM81xNJ+aSVizH61evCS+s40l0q5jhWYDy/2oGCMgxsc9gwAPtUzx +F4ZTDQouJeGmm1TRp4xMvNHyzRoRnJHRh7j7Vmmr+ZYTW+q2p/aQNhwD+de6mojWPETxMvdE0KG1 +1LmFxFzJDaKRGWYE/E6rsFHQD296wHiDjbiTXJna71KZUb/8UR5Fx6bdfrTXiC9u9b1iW/uA3NId +lySFHTApyugO9mLiHJI3xVVXzzN8TMSfc0406/u9OnE9pO8TjuO9WCLR47q3DACGdR3GVb2P96tX +iN4K8Q8G8LR8QXV5YzxYX8RBEx5oS3pnZgDtQJaD4j3z2fk6laC5jGzSR/mX3q26JxClzah47kTI +35SVwe+29UfwY0NNQur2/uA4ihVY0IPVzv8AI7Dv61aNStX03U5YT5fKeZ4+VcdOQ7j1xn7VApqL +/wCZWNzbSr+zmjZcHsazC202WwnVortlukJ/J0BHbPetK06QvEzEHIkI+hFI3fh1+O4e1Pia31uO +G5t5ARZsm7DAyQc5/Sg0Dwd4ztdU4Y/C6jMyX1rJycnLn4NsfYnGPf2q6aVq2m6tC02n3ccyI5jf +BwUYdQwO4Psa8kcLa3LpGssWeTyXfEgU4Jwa2vWLxdQ4e1oWGnabc6Zcwfio721YJcQuF5v2i7MQ +G2BGwzvRZWsOuKTIIqgeC/FjaxpB0q+maS+tF+FnOWkj9fmOn2rQGO9ZsUl0augkVwjfOKAB9qiu +52pNmzttRiDRGU5xQGQ9xQrqKcbihWaIxF70snWklB74paJcmtQRXHHEq8NaA90ih7uU8lup9cbs +fYf2rDLvWJ9QvGur6WWWd2y7Mc5q9eOV60Op6XbqhYLA8jHtu2P6VQYruAkeZFsepxW2KfQTI35G +Gau/DvG99aWo0zW401vR2xz211livujHdSO3p7VSLdrGX9/lPTI9KXaCXk/ZOjemaDW9M02NpbTi +LT/FjVrPhaxbzf8AL7mTnmgYDaH4iQVxtuDt69aoviLrXD+u67LcaLpE1pBJnzWcgLMf4gmAVPr2 +/nVMnlBcRXCBJ13QkbA06065FxCWcYdG5WT0IoGI0eISMgUe2RR9OJt5Ws3xjtt1qUB518zG6HtU +frScskVygIwd8UHZ7copUCjaxe8bcR6Yuj3V7NeW5OUiCc0jY3+ZG2aXibmUSb4YdRU3wbxqvA2q +zak1i9z5kBjDkcxjOf5f2FTQXwqnWz4eOmvGY5orhxICuG5tuvv2ol9rcOs69d+UcpZOsRYfvFld +f54qNtOJ5+Jtf1XVpMwySzI/L3/Ly/8A1FHg0aLS9aklslKw3flM8Z3AbzR09tzVEjYcoeULsGAc +CmvEOnarqt1aWelreSySK3PBbgnnUYO4HYUdJkSWAKd+XlOPepJ+J9S4VhttZ01Od1k5JEJ/OpBy +PuB9hQZLxHpk+n6sySQvE2fiR1wVYdQQad2F5eWkb/h5XiJjaNgDsVbZhj3qa4o1W64nu5tVvIDF +M7c2CcmohlBJ2xlaB9wzrl1w/r8Go2ZUyxDHK35WB6g16R4T1634i0SHUYF8st8MkZOeRh1FeW/J +WRw2enU1qvgnxTp+nR3Gj6lMtuZ5Q8EjbKSRjlJ7dBUpK2QkD0rg3zRGIJ2O1GSstlFwAc0VsA7V +35UXBzUo6p2oV0KaFZVGKm9LxKM9M0T0NdklENvLLjPIhYD5CukRjfi1qCXHF88eQY7ZFgH2yf1J ++1VyGOORMYGKbNK97JPLcMWkkcyOT3J60haieGQqmeXO2a0wkJ7FGHwkKexBpBLqeycJOTy9m7U/ +tJo5sebGMinM8VpLGedMqN996COvF/Hw+ZFyyOo6A9fl71H2NwEvg2cGQcrjpuOmf+u1PxZRK7SW +NyFx+5UfqyK3/eEBW4jPM4/jH96Im7d/jOGG/auXo57RlO/ptTHS7xJ4lIOT0p60mQ4GxopppdwG +QwOfy7UtIshlFuInmD4AULnOegqOGI7wMo2JwfSn8mXQgNysR8LZ70Fp4h8ML/hWK01C51Czb8eq +q0ETEtGx3HsR7im96wjWJFlVmjKq3fmxlz/8ahtK1TXdc4jjt9cvGljit2EQXbmIAAz9KXnYWVq8 +apjCSSDfJGwTr/yNAiMlFy/7oORvjepDihUk0CGFZFjDToDI/wCUZOMn7060vT7OXQVvpLadCwZY +wzBEkPL8JJPQBs5OcHp61E8Xs3+TwW867mQcy47gGgl+P+CtL4d0qyv9L1w6glwgLq6gEZHUY/lv +Wb3MmG5c4PKR9akYbaRinPczyRpuiO2QufTNRuqxNHdKvYsD/wBfagPFtEAegFFkbzcKq7A9aDDK +igx8qHb8zHCiiPQfhJrX+bcJQiWQyT2rGFyeu3Q/aropX0rzp4e8YS8JXqxPCJ7O5I89R+ZcfvCv +QOnXttqFlFeWkgkglUMjDuKzY1DvPegGoLRWqNFFbI7UK4g2O1CshgAcelGRFkUxt0YEH610jAo0 +LKD71qDzaI1g1Ke1JB5XZM/I4o1xGY2Db4FLcWRfgOML9OgS8kH0LH+4p75AmgDcwwa2wYYLplNi +BtiixzzRH4gWA7EVyM/h5miz8NPIRHMOUAADvnFENpYrO7X87wSnoU/tTC80bUoR5kEgvUG+xww+ +h/pUxJawr8cec02FxJE27HlHrQQOkTG2lliZCuGyA2xHtUxFKzfER9qYaxBLPci9tD5g5cSRj8w9 +x6ijW8+YsZHTegPLvuMDBp2JAIlbbcY6U2tpPMV8/LOKAceXjB2oJ3hCeN9Z3UMywvjbp0rt9bmS +OeeQZeQQrGN8AkFjt26ioLhed4uKIU5iOdXBH/GrLqDfEkaDc3BJ/wCKqv8ASirgbmxtLENZ6W7x +ae37cMQFYc+Qq8wOSAASfXvvVP8AEdndhLJzeZ+LfmBbJyc7VJPdW0ziW6hilkAzzFFBHyqJ8RLt +brTLec8gkacFiFwT8JoIK2lXlBYbVE8RMFaKT19/ejG5wAq0x1uVnhiB3IbagdwqHVc9MUSP9vdF +v3E+FaPptvdX7Q2NhDJPdzEJHGgySTWq2/hOukcMvd6teSC/EZZUhwY1OM4JxknttjrQZqYlDGRx +sOgNaX4M8Sw2Usmi6hOscUp5rcucAN3X60Xh3wf4k1e1ju7ua105GHMsUuWk/wCQH5flnNUnUrFo +NVms5MB7VzGeX+JTg4+1QemVwRkYxRGJzWd+H3HtrJDBo+sytDdKOSOdz8MnoCexrRRg4IIIPTFZ +aHRtutCiEkChWappJkLjJrkaEnJo8rCgjYxvW4MN8XrNYeNr3GR5ipJ09VFRWg3XnQlMnmTYg1dP +G+z/APGbO8HSa25D81J/oRWXQztYXvm78j7PVYqw6hFGSJOXp1OKbpzKOZExipGIxXduGT4sjam3 +lmJzG+du4NULW84YBZCPfaiXtosykxt13xXFCEEDOR39aNDKUk/KceuelEVq/gu4JcRrICD13okr +XBhE8x+IHDEdSOxNW55oHT4lXPqKjr+2je0kjHLlxjtRURZNiPYjfvXWbAIYCkbZuVeQnp1HvSU8 +yDIBG+2KIW055DxDp7W7AP5uCcA/Dj4v0zVnhdpZIWOxMZkP/Ni230IqrcMRo2tyTqCTDbyMBnbJ +HKP/AJVaLGRBcXDkhkiYRqADsFGB/KosSyxQOg50JPuKqvH7CJrG2RsjDyN+gFS9veTT3W4+HPrV +X48uObWljBwI4gMDsTRTbhvTbjXtfstHtZooprqURh5Gwq+5qa8ZeCLjgi6s7d9VttRSUkiSIcpV +h1BXP61VrW4ltikttIYZo2DI69Qw70e9utS1/VrSPULlp5ZZVXJwOpAJrSN7/wAO3B6Wul/9qL5R ++JuU5bcMu6Reo92/lWo2GmS6hqBv78AJGSLe3JyFH8Te/f2+dNOHEkt+FtOtRcRmNY1AePBGw2pT +U9RFoHSW9htI+txcu3KIl74z1JOwHr8qjUTkY/Fo0Vs7RW6bM46k98e9Y54+No6alYWtjbRw3MMZ +82RVwFQ9Ax9Sd9/ep3WvFzQ7G3FhollcXap8IkJ8tSPXfcn3xVP408QdP4k4Yk0M6GLbmkEvmGfn +JcHOTsPehqhpb2kjiTl53H72dqv/AADxs9hyaZquTaDaObmyY/Y+1Z000iAsRGF6DkJOPnSE1xME +ODzKR2rNiPTqyxTQrLDIskbDKspyCKFZ14IX7XGgXFm0vP5MpIBOcA9PpQrNaXlmwBtXRkkGkQwz +SqZyKSireMVh+I4TivAPitJ1JP8ApbY/ry1h+pW3OTgnHbevQPHsN3qOjwaDYcguNTnEILdAoBcn +/wBtYMRzKFIww2atxmmGl6nPps3lOeaPO4qz2t5aXyghwD71WLy18zJC71FwzSW8uCSKrK+XFpyn +nifJPXemp8xdgjZ+RqOtLucqG5+YEbUs2pyoCWJoHEqyn8sbkkdlNRV4uoKwPkuvtykVIwcQRK2G +yKPNrEEzpySFGXpk/Cw9DRVT1qeW1lWQoyCUZx03HWo1LtpG3JrdvBySz1fxAtbDUeG7PUomVjI8 +8SukAxs45ts5GPXrXp+z4b4chAaHQ9MjI6FbVB/SkMeG/CzTdS1jiRNOtdPu5TduiGZISyRKrcxZ +j6bV6l4a8FuDLSxWO7hvb+U7vJNM0ZLHr8KYx+taqkcUIVI41VewUYArsigSZGKuKoKeEHAMJDpo +rhvX8VKf5tTW78DfDW+na5udDleV8Zb8ZMOnyatLyCu/pQQgDqKDNbPwJ8M7W6juU4e8xozkLLcy +un1Utg/WlX8EfDsasurW+hm3u0bmUxXEiqp9lzj9K0gkAda7nK7VRVZeDdIeJwqzwzH8s6vzPGfU +Bsr+lecPGPh/ifh/iBbLW9Sl1GxkzJY3LfCGGdwR0DDv8/evWb7HOdqpvjLwsvGHh9qGnxIDf26G +4sX7iVRnl/5br9amFeQiOQ4yCPbrTeSLLl4zv1371DQX8yA+YCGHX2NBuJCshR0UgbdKiHdxd8jl +dwfSm348xnm5Oai/ibe/bflRj0P96ZXHNBLyMpA96GtG8G9cS34rWDIWO9Qxuv8AqG6n+n1oVV/D +S3a7490qNCwVZfNbHooJ/pQrnZjUejBjPSl12IG1Nhkt0pxGD1NSVcQ3F17HpeqcOalNJyRRaj5b +tnAAdGWsIlYJql7CSMpcOvX0Y1vvG3DZ4q4Yn0qO4W3nLLJDKwyEZTnt7ZFYxxfwJq3DV9LNNfR6 +j+xWeeRFK8nM3IOvXfG9bjNRbJzHAxnGMUwurJJVbb608t5gFywJJ9KcZXBBGRVREabM1rP5E26n +cGp0Q288PMNjTC/tkki5kwCB2613SblsCF2APT50AvNMj5TIuB3z3qNSMBvi7Hap65TbP9dqirhV +QFsYOelNMW7wn1ltC4uspjPy28kgSTmOwB6H23r2Fpl+rwhieY8ucj+9fP1794mJBAxWhcEeJHF1 +vYCCDU3Fra/xgNk9lyRnHtRXtA3S8iuCppvd38ceDg7HtWD6B40W7Wwe9tQZlHxRBsBv9p7VcND4 +80ni+ynbTGkt7222ntJtnXPQjsw9xV1V8XX7U8o5XO5H8v70dNcsOflZ+VvQmszvG1Yc7RJzxk5y +N+tIQX8sbjmkhBO551II+9T9GNfTUbaTHJMjfI04S4UjGay+zvp5CpJBP+kbCp2y1GRByFmPfc9K +foxb2lDE7/ShDIomUZ2OxHzquG8nZQyinNpdyNgEYYGmmPH3jLoFpo3iVrulEGBBcmaEqNuST4wP +pzY+lUu60i2EBImVpPbvWwf4z7Q23iBpeoRLgXenAE+pR2/owrCo7iUOAT9KrIvJJbyfDkYNSkFx +Hd24jlxzr+U0QeVdDHMFIG5NMJk8p/gl+1X0aj4Eaf5vFNzeFdrW2IB92IA/QGhVn8BdNntOF59U +uYyn46UeVnqyLtn5Ek/ahXLq/Wo0LlAbY0dAc9TSJOD70ojjY1iNU7iyM4yKqfF8IvNa1KyfcT8N +3DKD3aN1dftirVDKp9Kz/wAbLq60pdO1ywmWN+SazfftIvp8s1uJWNluQq/NsRnrTtJkbBUjGKj7 +OSOeE2rN8QGVNNj51jNljzRnvW2U7JKrKQoyR1phM/lSK64z1NKQOsq8/MObrmkJjzty8h9M1BOW +E63UIQMOalZdBkul+J8DtvVTE01pLzwyMuPQU4bi3UUQJkEjbJFMNSl1wzbwjnurxQvcLTC61OGO +FdO08IkS+nf5moDUtbvr5irynHtSNoxiIbO9WRFnVWEBfm3A9abadJxE16l3pN7cWk8eyyxyFDj0 +yO3tSNpqSD4ZcYPWp231e0t7blgYA43FBYLbxG474Zt+eW/ttULYDrOhJP1BGftVguOK+MNV4cte +I7LheG4sp8q01rO+Y5FPxKy9j36dMVlVzfSX1zsMhdzWm/4fePLPhniSTh7XJUj0bWCFLufggnH5 +XPoD+Un5HtRZVq4J8Q+GDpbza/FrMFzAc3Kxwq/kL/ER1K57gbbVpPDHFPAuuMo0jjbT5XbcQzv5 +T/8Apf8AtTTirwj0niC9N7a6oulXirmKSABiT/qGd1x2715q8SOAZ9D4gm0y4jWw1INlMHFtdL2a +Mn8pP8JqZFe1raxkKjlaGZT3RhinSWLKdosZr532mu8UcPXLQ2uralp8sZwViuHT+Rqaj8XPEeKE +xJxfqnKRjeXJ+53rWJrYv8cd9YC94ask5Wv44pXch90jJUAEe5B+1eaDKSMgkGlNV1HUNVvZL3Ur +ye7uZDl5ZnLMfqaluDODeIeLLvydHsHkjU4kuH+GKP5sf5DJ9qvgghI4YkM2/vWreEXhhfcQSxaz +r8Ulto6kMkbfC917DuF9+/b1GkcB+EPDvDix3mphdX1Jfi5pF/Yxn/Snf5tn5Cr9PP22A6Vi9Lhp +NHFEkcEEaxRRqFRFGAoHQAUKTnkHPmhXOqTcnNBGNChWI0VjbvgVnX+IiMtwbZzKQDHeA/dWoUK6 +c+s1g0N2/mrMuzKas0YS/tQ7LguvN8qFCt1kwV2tJ/KU5XNO2cqSQBkj0oUKBheFmzvudyah51Oe +tChViVyOFQfenCxKRvQoVUJSRgHrSTMQ2ATQoVYsOIZ3gBAJpve3Ek5BJ70KFIJXh/i/ibQZEl0r +Wbu2KkEKJCV/9J2q73Pi3xPxRZ/5JqENhNPe8tsLqSLLICcbD60KFLCM+4gDw63cWUkjTpaSNApb +qQpIrWPD/wAMeG+IuDLW/vWvYbqUNl4ZRjrtsQaFCs9XIs9XHhvwa4M04CS9hudVl5sg3EnKg/4r +jP1zWhW8dvY2kdpZW8NtbxjCRRIFVR7AUKFc9tawSSViDTSRsnehQqKbyjLbUKFCs0j/2Q== + +Cheerio! + +--Multipart_Sun_Oct_17_10:37:40_2010-1-- diff --git a/testdata/testdir2/Foo/new/.noindex b/testdata/testdir2/Foo/new/.noindex new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/Foo/tmp/.noindex b/testdata/testdir2/Foo/tmp/.noindex new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/bar/.noupdate b/testdata/testdir2/bar/.noupdate new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/bar/cur/181736.eml b/testdata/testdir2/bar/cur/181736.eml new file mode 100644 index 0000000..56255c4 --- /dev/null +++ b/testdata/testdir2/bar/cur/181736.eml @@ -0,0 +1,42 @@ +Path: uutiset.elisa.fi!feeder2.news.elisa.fi!feeder.erje.net!newsfeed.kamp.net!newsfeed0.kamp.net!nx02.iad01.newshosting.com!newshosting.com!post01.iad!not-for-mail +X-newsreader: xrn 9.03-beta-14-64bit +Sender: jimbo@lews (Jimbo Foobarcuux) +From: jimbo@slp53.sl.home (Jimbo Foobarcuux) +Reply-To: slp53@pacbell.net +Subject: Re: Are writes "atomic" to readers of the file? +Newsgroups: comp.unix.programmer +References: <87hbblwelr.fsf@sapphire.mobileactivedefense.com> <8762s0jreh.fsf@sapphire.mobileactivedefense.com> <87hbbjc5jt.fsf@sapphire.mobileactivedefense.com> <8ioh48-8mu.ln1@leafnode-msgid.gclare.org.uk> +Organization: UseNetServer - www.usenetserver.com +X-Complaints-To: abuse@usenetserver.com +Message-ID: +Date: 08 Mar 2011 17:04:20 GMT +Lines: 27 +Xref: uutiset.elisa.fi comp.unix.programmer:181736 + +John Denver writes: +>Eric the Red wrote: +> +>>> There _IS_ a requirement that all reads and writes to regular files +>>> be atomic. There is also an ordering guarantee. Any implementation +>>> that doesn't provide both atomicity and ordering guarantees is broken. +>> +>> But where is it specified? +> +>The place where it is stated most explicitly is in XSH7 2.9.7 +>Thread Interactions with Regular File Operations: +> +> All of the following functions shall be atomic with respect to each +> other in the effects specified in POSIX.1-2008 when they operate on +> regular files or symbolic links: +> +> [List of functions that includes read() and write()] +> +> If two threads each call one of these functions, each call shall +> either see all of the specified effects of the other call, or none +> of them. +> + +And, for the purposes of this paragraph, the two threads need not be +part of the same process. + +jimbo diff --git a/testdata/testdir2/bar/cur/mail1 b/testdata/testdir2/bar/cur/mail1 new file mode 100644 index 0000000..56808c6 --- /dev/null +++ b/testdata/testdir2/bar/cur/mail1 @@ -0,0 +1,38 @@ +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "John Milton" +Subject: Fere libenter homines id quod volunt credunt +To: "Julius Caesar" +Message-id: <3BE9E6535E3029448670913581E7A1A20D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +x-label: Paradise losT +X-Keywords: milton,john +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +OF Mans First Disobedience, and the Fruit +Of that Forbidden Tree, whose mortal tast +Brought Death into the World, and all our woe, +With loss of Eden, till one greater Man +Restore us, and regain the blissful Seat, [ 5 ] +Sing Heav'nly Muse,that on the secret top +Of Oreb, or of Sinai, didst inspire +That Shepherd, who first taught the chosen Seed, +In the Beginning how the Heav'ns and Earth +Rose out of Chaos: Or if Sion Hill [ 10 ] +Delight thee more, and Siloa's Brook that flow'd +Fast by the Oracle of God; I thence +Invoke thy aid to my adventrous Song, +That with no middle flight intends to soar +Above th' Aonian Mount, while it pursues [ 15 ] +Things unattempted yet in Prose or Rhime. +And chiefly Thou O Spirit, that dost prefer +Before all Temples th' upright heart and pure, +Instruct me, for Thou know'st; Thou from the first +Wast present, and with mighty wings outspread [ 20 ] +Dove-like satst brooding on the vast Abyss +And mad'st it pregnant: What in me is dark +Illumin, what is low raise and support; +That to the highth of this great Argument +I may assert Eternal Providence, [ 25 ] +And justifie the wayes of God to men. diff --git a/testdata/testdir2/bar/cur/mail2 b/testdata/testdir2/bar/cur/mail2 new file mode 100644 index 0000000..3799f30 --- /dev/null +++ b/testdata/testdir2/bar/cur/mail2 @@ -0,0 +1,14 @@ +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "Socrates" +Subject: cool stuff +To: "Alcibiades" +Message-id: <3BE9E6535E0D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +The hour of departure has arrived, and we go our ways—I to die, and you to +live. Which is better God only knows. + +http-emacs diff --git a/testdata/testdir2/bar/cur/mail3 b/testdata/testdir2/bar/cur/mail3 new file mode 100644 index 0000000..646365e --- /dev/null +++ b/testdata/testdir2/bar/cur/mail3 @@ -0,0 +1,34 @@ +From: Napoleon Bonaparte +To: Edmond =?UTF-8?B?RGFudMOocw==?= +Subject: rock on dude +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +Fcc: .sent +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Le 24 février 1815, la vigie de Notre-Dame de la Garde signala le trois-mâts +le Pharaon, venant de Smyrne, Trieste et Naples. + +Comme d'habitude, un pilote côtier partit aussitôt du port, rasa le château +d'If, et alla aborder le navire entre le cap de Morgion et l'île de Rion. + +Aussitôt, comme d'habitude encore, la plate-forme du fort Saint-Jean s'était +couverte de curieux; car c'est toujours une grande affaire à Marseille que +l'arrivée d'un bâtiment, surtout quand ce bâtiment, comme le Pharaon, a été +construit, gréé, arrimé sur les chantiers de la vieille Phocée, et appartient +à un armateur de la ville. + +Cependant ce bâtiment s'avançait; il avait heureusement franchi le détroit que +quelque secousse volcanique a creusé entre l'île de Calasareigne et l'île de +Jaros; il avait doublé Pomègue, et il s'avançait sous ses trois huniers, son +grand foc et sa brigantine, mais si lentement et d'une allure si triste, que +les curieux, avec cet instinct qui pressent un malheur, se demandaient quel +accident pouvait être arrivé à bord. Néanmoins les experts en navigation +reconnaissaient que si un accident était arrivé, ce ne pouvait être au +bâtiment lui-même; car il s'avançait dans toutes les conditions d'un navire +parfaitement gouverné: son ancre était en mouillage, ses haubans de beaupré +décrochés; et près du pilote, qui s'apprêtait à diriger le Pharaon par +l'étroite entrée du port de Marseille, était un jeune homme au geste rapide et +à l'œil actif, qui surveillait chaque mouvement du navire et répétait chaque +ordre du pilote. diff --git a/testdata/testdir2/bar/cur/mail4 b/testdata/testdir2/bar/cur/mail4 new file mode 100644 index 0000000..4d21a48 --- /dev/null +++ b/testdata/testdir2/bar/cur/mail4 @@ -0,0 +1,29 @@ +Return-Path: +Delivered-To: foo@example.com +Received: from [128.88.204.56] by freemailng0304.web.de with HTTP; + Mon, 07 May 2005 00:27:52 +0200 +Date: Mon, 07 May 2005 00:27:52 +0200 +Message-Id: <293847329847@web.de> +MIME-Version: 1.0 +From: =?iso-8859-1?Q? "=F6tzi" ?= +To: foo@example.com +Subject: =?iso-8859-1?Q?Re:=20der=20b=E4r=20und=20das=20m=E4dchen?= +Precedence: fm-user +Organization: http://freemail.web.de/ +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: 8bit +X-MIME-Autoconverted: from quoted-printable to 8bit by mailhost6.ladot.com id j48MScQ30791 +X-Label: \backslash +X-UIDL: 93h!!\i + +123 diff --git a/testdata/testdir2/bar/cur/mail6 b/testdata/testdir2/bar/cur/mail6 new file mode 100644 index 0000000..c9b799b --- /dev/null +++ b/testdata/testdir2/bar/cur/mail6 @@ -0,0 +1,18 @@ +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "Geoff Tate" +Subject: eyes of a stranger +To: "Enrico Fermi" +Message-id: <3BE9E6535E302944823E7A1A20D852173@msg.id> +MIME-version: 1.0 +X-label: @NextActions operation:mindcrime Queensrÿche +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +And I raise my head and stare +Into the eyes of a stranger +I've always known that the mirror never lies +People always turn away +From the eyes of a stranger +Afraid to know what +Lies behind the stare diff --git a/testdata/testdir2/bar/cur/mail7 b/testdata/testdir2/bar/cur/mail7 new file mode 100644 index 0000000..48660f3 --- /dev/null +++ b/testdata/testdir2/bar/cur/mail7 @@ -0,0 +1,16 @@ +Date: Mon, 11 Sep 2023 19:57:25 -0400 +From: "Tommy" +Subject: Hide and seek +To: "Andreas" +Message-id: <3BE9E65sdfklsajdfl3E7A1A20D852173@msg.id> +MIME-version: 1.0 + +Behind the polished barrier +The pending storm draws near +Although it's an inferno +You can not step back for your fears +Hurry son I need to rest +finish the puzzle you do it best +I'll do what I can +But I am telling you +It can't be done without you! diff --git a/testdata/testdir2/bar/new/.noindex b/testdata/testdir2/bar/new/.noindex new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/bar/tmp/.noindex b/testdata/testdir2/bar/tmp/.noindex new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/wom_bat/cur/atomic b/testdata/testdir2/wom_bat/cur/atomic new file mode 100644 index 0000000..c3c6792 --- /dev/null +++ b/testdata/testdir2/wom_bat/cur/atomic @@ -0,0 +1,20 @@ +Date: Sat, 12 Nov 2011 12:06:23 -0400 +From: "Richard P. Feynman" +Subject: atoms +To: "Democritus" +Message-id: <3BE9E6535E302944823E7A1A20D852173@msg.id> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +If, in some cataclysm, all scientific knowledge were to be destroyed, +and only one sentence passed on to the next generation of creatures, +what statement would contain the most information in the fewest words? +I believe it is the atomic hypothesis (or atomic fact, or whatever you +wish to call it) that all things are made of atoms — little particles +that move around in perpetual motion, attracting each other when they +are a little distance apart, but repelling upon being squeezed into +one another. In that one sentence you will see an enormous amount of +information about the world, if just a little imagination and thinking +are applied. diff --git a/testdata/testdir2/wom_bat/cur/rfc822.1 b/testdata/testdir2/wom_bat/cur/rfc822.1 new file mode 100644 index 0000000..71c3107 --- /dev/null +++ b/testdata/testdir2/wom_bat/cur/rfc822.1 @@ -0,0 +1,44 @@ +Return-Path: +Subject: Fwd: rfc822 +From: foobar +To: martin +Content-Type: multipart/mixed; boundary="=-XHhVx/BCC6tJB87HLPqF" +Message-Id: <1077300332.871.27.camel@example.com> +Mime-Version: 1.0 +X-Mailer: Ximian Evolution 1.4.5 +Date: Fri, 20 Feb 2004 19:05:33 +0100 + +--=-XHhVx/BCC6tJB87HLPqF +Content-Type: text/plain +Content-Transfer-Encoding: 7bit + +Hello world, forwarding some RFC822 message + +--=-XHhVx/BCC6tJB87HLPqF +Content-Disposition: inline +Content-Type: message/rfc822 + +Return-Path: +Message-ID: <9A01B19D0D605D478E8B72E1367C66340141B9C5@example.com> +From: frob@example.com +To: foo@example.com +Subject: hopjesvla +Date: Sat, 13 Dec 2003 19:35:56 +0100 +MIME-Version: 1.0 +Content-Type: text/plain; charset=iso-8859-1 +Content-Transfer-Encoding: 7bit + +The ship drew on and had safely passed the strait, which some volcanic shock +has made between the Calasareigne and Jaros islands; had doubled Pomegue, and +approached the harbor under topsails, jib, and spanker, but so slowly and +sedately that the idlers, with that instinct which is the forerunner of evil, +asked one another what misfortune could have happened on board. However, those +experienced in navigation saw plainly that if any accident had occurred, it was +not to the vessel herself, for she bore down with all the evidence of being +skilfully handled, the anchor a-cockbill, the jib-boom guys already eased off, +and standing by the side of the pilot, who was steering the Pharaon towards the +narrow entrance of the inner port, was a young man, who, with activity and +vigilant eye, watched every motion of the ship, and repeated each direction of +the pilot. + +--=-XHhVx/BCC6tJB87HLPqF-- diff --git a/testdata/testdir2/wom_bat/cur/rfc822.2 b/testdata/testdir2/wom_bat/cur/rfc822.2 new file mode 100644 index 0000000..316fa3f --- /dev/null +++ b/testdata/testdir2/wom_bat/cur/rfc822.2 @@ -0,0 +1,44 @@ +From: dwarf@siblings.net +To: root@eruditorum.org +Subject: Fwd: test abc +References: <8639ddr9wu.fsf@cthulhu.djcbsoftware> +User-agent: mu 0.98pre; emacs 24.0.91.9 +Date: Thu, 24 Nov 2011 14:24:00 +0200 +Message-ID: <861usxr9nj.fsf@cthulhu.djcbsoftware> +Content-Type: multipart/mixed; boundary="=-=-=" +MIME-Version: 1.0 + +--=-=-= +Content-Type: text/plain + +Saw the website. Am willing to stipulate that you are not RIST 9E03. Suspect +that you are the Dentist, who yearns for honest exchange of views. Anonymous, +digitally signed e-mail is the only safe vehicle for same. + +If you want me to believe you are not the Dentist, provide plausible +explanation for your question regarding why we are building the Crypt. + +Yours truly, + +--=-=-= +Content-Type: message/rfc822 +Content-Disposition: inline; filename= + "1322137188_3.11919.foo:2,S" +Content-Description: rfc822 + +From: dwarf@siblings.net +To: root@eruditorum.org +Subject: test abc +User-agent: mu 0.98pre; emacs 24.0.91.9 +Date: Thu, 24 Nov 2011 14:18:25 +0200 +Message-ID: <8639ddr9wu.fsf@cthulhu.djcbsoftware> +Content-Type: text/plain +MIME-Version: 1.0 + +As I stepped on this unknown middle-aged Filipina's feet during an ill-advised +ballroom dancing foray, she leaned close to me and uttered some latitude and +longitude figures with a conspicuously large number of significant digits of +precision, implying a maximum positional error on the order of the size of a +dinner plate. Gosh, was I ever curious! + +--=-=-=-- diff --git a/testdata/testdir4/1220863042.12663_1.mindcrime!2,S b/testdata/testdir4/1220863042.12663_1.mindcrime!2,S new file mode 100644 index 0000000..ab1500f --- /dev/null +++ b/testdata/testdir4/1220863042.12663_1.mindcrime!2,S @@ -0,0 +1,146 @@ +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-4.9 required=3.0 tests=BAYES_00,DATE_IN_PAST_96_XX, + RCVD_IN_DNSWL_MED autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 5123469CB3 + for ; Thu, 7 Aug 2008 08:10:19 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for (single-drop); Thu, 07 Aug 2008 08:10:19 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs39272wfh; Wed, 6 Aug 2008 + 20:15:17 -0700 (PDT) +Received: by 10.65.133.8 with SMTP id k8mr2071878qbn.7.1218078916289; Wed, 06 + Aug 2008 20:15:16 -0700 (PDT) +Received: from sourceware.org (sourceware.org [209.132.176.174]) by + mx.google.com with SMTP id 28si7904461qbw.0.2008.08.06.20.15.15; Wed, 06 Aug + 2008 20:15:16 -0700 (PDT) +Received-SPF: neutral (google.com: 209.132.176.174 is neither permitted nor + denied by domain of gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org) + client-ip=209.132.176.174; +Authentication-Results: mx.google.com; spf=neutral (google.com: + 209.132.176.174 is neither permitted nor denied by domain of + gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org) + smtp.mail=gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org +Received: (qmail 13493 invoked by alias); 7 Aug 2008 03:15:13 -0000 +Received: (qmail 13485 invoked by uid 22791); 7 Aug 2008 03:15:12 -0000 +Received: from mailgw1a.lmco.com (HELO mailgw1a.lmco.com) (192.31.106.7) + by sourceware.org (qpsmtpd/0.31) with ESMTP; Thu, 07 Aug 2008 03:14:27 +0000 +Received: from emss07g01.ems.lmco.com (relay5.ems.lmco.com [166.29.2.16])by + mailgw1a.lmco.com (LM-6) with ESMTP id m773EPZH014730for + ; Wed, 6 Aug 2008 21:14:25 -0600 (MDT) +Received: from CONVERSION2-DAEMON.lmco.com by lmco.com (PMDF V6.3-x14 #31428) + id <0K5700601NO18J@lmco.com> for gcc-help@gcc.gnu.org; Wed, 06 Aug 2008 + 21:14:25 -0600 (MDT) +Received: from EMSS04I00.us.lmco.com ([166.17.13.135]) by lmco.com (PMDF + V6.3-x14 #31428) with ESMTP id <0K5700H5MNNWGX@lmco.com> for + gcc-help@gcc.gnu.org; Wed, 06 Aug 2008 21:14:20 -0600 (MDT) +Received: from EMSS35M06.us.lmco.com ([158.187.107.143]) by + EMSS04I00.us.lmco.com with Microsoft SMTPSVC(5.0.2195.6713); Wed, 06 Aug + 2008 23:14:20 -0400 +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "Mickey Mouse" +Subject: gcc include search order +To: "Donald Duck" +Message-id: <3BE9E6535E3029448670913581E7A1A20D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Content-class: urn:content-classes:message +Mailing-List: contact gcc-help-help@gcc.gnu.org; run by ezmlm +Precedence: klub +List-Id: +List-Unsubscribe: +List-Archive: +List-Post: +List-Help: +Sender: gcc-help-owner@gcc.gnu.org +Delivered-To: mailing list gcc-help@gcc.gnu.org +Content-Length: 3024 + + +Hi. +In my unit testing I need to change some header files (target is +vxWorks, which supports some things that the sun does not). +So, what I do is fetch the development tree, and then in a new unit test +directory I attempt to compile the unit under test. Since this is NOT +vxworks, I use sed to change some of the .h files and put them in a +./changed directory. + +When I try to compile the file, it is still using the .h file from the +original location, even though I have listed the include path for +./changed before the include path for the development tree. + +Here is a partial output from gcc using the -v option + +GNU CPP version 3.1 (cpplib) (sparc ELF) +GNU C++ version 3.1 (sparc-sun-solaris2.8) + compiled by GNU C version 3.1. +ignoring nonexistent directory "NONE/include" +#include "..." search starts here: +#include <...> search starts here: + . + changed + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/mp/interface + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/ap/app + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/interface + /usr/local/include/g++-v3 + /usr/local/include/g++-v3/sparc-sun-solaris2.8 + /usr/local/include/g++-v3/backward + /usr/local/include + /usr/local/lib/gcc-lib/sparc-sun-solaris2.8/3.1/include + /usr/local/sparc-sun-solaris2.8/include + /usr/include +End of search list. + +I know the changed file is correct and that the include is not working +as expected, because when I copy the file from ./changed, back into the +development tree, the compilation works as expected. + +One more bit of information. The source that I cam compiling is in +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/ap/app +And it is including files from +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common +These include files should be including the files from ./changed (when +they exist) but they are ignoring the .h files in the ./changed +directory and are instead using other, unchanged files in the +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common +directory. + +The gcc command line is something like + + TEST_DIR="." + + CHANGED_DIR_NAME=changed + CHANGED_FILES_DIR=${TEST_DIR}/${CHANGED_DIR_NAME} + + CICU_HEADER_FILES="-I ${AP_INTERFACE_FILES} -I ${AP_APP_FILES} -I +${SHARED_COMMON_FILES} -I ${SHARED_INTERFACE_FILES}" + + HEADERS="-I ./ -I ${CHANGED_FILES_DIR} ${CICU_HEADER_FILES}" + DEFINES="-DSUNRUN -DA10_DEBUG -DJOETEST" + + CFLAGS="-v -c -g -O1 -pipe -Wformat -Wunused -Wuninitialized -Wshadow +-Wmissing-prototypes -Wmissing-declarations" + + printf "Compiling the UUT File\n" + gcc -fprofile-arcs -ftest-coverage ${CFLAGS} ${HEADERS} ${DEFINES} +${AP_APP_FILES}/unitUnderTest.cpp + + +I hope this explanation is clear. If anyone knows how to fix the command +line so that it gets the .h files in the "changed" directory are used +instead of files in the other include directories. + +Thanks +Joe + +---------------------------------------------------- +Time Flies like an Arrow. Fruit Flies like a Banana + + diff --git a/testdata/testdir4/1220863087.12663_19.mindcrime!2,S b/testdata/testdir4/1220863087.12663_19.mindcrime!2,S new file mode 100644 index 0000000..78efa2a --- /dev/null +++ b/testdata/testdir4/1220863087.12663_19.mindcrime!2,S @@ -0,0 +1,77 @@ +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id C4D6569CB3 + for ; Thu, 7 Aug 2008 08:10:08 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for (single-drop); Thu, 07 Aug 2008 08:10:08 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs34794wfh; Wed, 6 Aug 2008 + 13:40:29 -0700 (PDT) +Received: by 10.100.33.13 with SMTP id g13mr1093301ang.79.1218055228418; Wed, + 06 Aug 2008 13:40:28 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d19si15908789and.17.2008.08.06.13.40.27; Wed, 06 Aug 2008 + 13:40:28 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:56316 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQpo3-0007Pc-Qk for + xxxx.klub@gmail.com; Wed, 06 Aug 2008 16:40:27 -0400 +From: anon@example.com +Newsgroups: gnu.emacs.help +Date: Wed, 6 Aug 2008 20:38:35 +0100 +Message-ID: +References: <55dbm5-qcl.ln1@news.ducksburg.com> + +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +X-Trace: individual.net bABVU1hcJwWAuRwe/097AAoOXnGGeYR8G1In635iFGIyfDLPUv +X-Orig-Path: news.ducksburg.com!news +Cancel-Lock: sha1:wK7dsPRpNiVxpL/SfvmNzlvUR94= + sha1:oepBoM0tJBLN52DotWmBBvW5wbg= +User-Agent: slrn/pre0.9.9-120/mm/ao (Ubuntu Hardy) +Path: news.stanford.edu!headwall.stanford.edu!newshub.sdsu.edu!feeder.erje.net!proxad.net!feeder1-2.proxad.net!feed.ac-versailles.fr!fu-berlin.de!uni-berlin.de!individual.net!not-for-mail +Xref: news.stanford.edu gnu.emacs.help:160868 +To: help-gnu-emacs@gnu.org +Subject: Re: Learning LISP; Scheme vs elisp. +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 417 +Lines: 11 + +On 2008-08-01, Thien-Thi Nguyen wrote: + +> warriors attack, felling foe after foe, +> few growing old til they realize: to know +> what deceit is worth deflection; +> such receipt reversed rejection! +> then their heavy arms, e'er transformed to shields: +> balanced hooked charms, ploughed deep, rich yields. + +Aha: the exercise for the reader is to place the parens correctly. +Might take me a while to solve this puzzle. + diff --git a/testdata/testdir4/1252168370_3.14675.cthulhu!2,S b/testdata/testdir4/1252168370_3.14675.cthulhu!2,S new file mode 100644 index 0000000..1e69622 --- /dev/null +++ b/testdata/testdir4/1252168370_3.14675.cthulhu!2,S @@ -0,0 +1,22 @@ +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.1.0 (2005-09-13) on mindcrime +X-Spam-Level: +Delivered-To: dfgh@floppydisk.nl +Message-ID: <43A09C49.9040902@euler.org> +Date: Wed, 14 Dec 2005 23:27:21 +0100 +From: Fred Flintstone +User-Agent: Mozilla Thunderbird 1.0.7 (X11/20051010) +X-Accept-Language: nl-NL, nl, en +MIME-Version: 1.0 +To: dfgh@floppydisk.nl +List-Id: =?utf-8?q?Example_of_List_Id?= +Subject: Re: xyz +References: <439C1136.90504@euler.org> <4399DD94.5070309@euler.org> <20051209233303.GA13812@gauss.org> <439B41ED.2080402@euler.org> <4399DD94.5070309@euler.org> <20051209233303.GA13812@gauss.org> <439A1E03.3090604@euler.org> <20051211184308.GB13513@gauss.org> +In-Reply-To: <20051211184308.GB13513@gauss.org> +X-Enigmail-Version: 0.92.0.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit +X-UIDL: T +To: Bilbo Baggins +Subject: Greetings from =?UTF-8?B?TG90aGzDs3JpZW4=?= +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +Fcc: .sent +Organization: The Fellowship of the Ring +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + + +Let's write some fünkÿ text +using umlauts. + +Foo. diff --git a/testdata/testdir4/1305664394.2171_402.cthulhu!2, b/testdata/testdir4/1305664394.2171_402.cthulhu!2, new file mode 100644 index 0000000..863f714 --- /dev/null +++ b/testdata/testdir4/1305664394.2171_402.cthulhu!2, @@ -0,0 +1,17 @@ +From: =?UTF-8?B?TcO8?= +To: Helmut =?UTF-8?B?S3LDtmdlcg==?= +Subject: =?UTF-8?B?TW90w7ZyaGVhZA==?= +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +References: +1n-Reply-To: +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + + +Test for issue #38, where apparently searching for accented words in subject, +to etc. fails. + +What about here? Queensrÿche. Mötley Crüe. + + diff --git a/testdata/testdir4/181736.eml b/testdata/testdir4/181736.eml new file mode 100644 index 0000000..56255c4 --- /dev/null +++ b/testdata/testdir4/181736.eml @@ -0,0 +1,42 @@ +Path: uutiset.elisa.fi!feeder2.news.elisa.fi!feeder.erje.net!newsfeed.kamp.net!newsfeed0.kamp.net!nx02.iad01.newshosting.com!newshosting.com!post01.iad!not-for-mail +X-newsreader: xrn 9.03-beta-14-64bit +Sender: jimbo@lews (Jimbo Foobarcuux) +From: jimbo@slp53.sl.home (Jimbo Foobarcuux) +Reply-To: slp53@pacbell.net +Subject: Re: Are writes "atomic" to readers of the file? +Newsgroups: comp.unix.programmer +References: <87hbblwelr.fsf@sapphire.mobileactivedefense.com> <8762s0jreh.fsf@sapphire.mobileactivedefense.com> <87hbbjc5jt.fsf@sapphire.mobileactivedefense.com> <8ioh48-8mu.ln1@leafnode-msgid.gclare.org.uk> +Organization: UseNetServer - www.usenetserver.com +X-Complaints-To: abuse@usenetserver.com +Message-ID: +Date: 08 Mar 2011 17:04:20 GMT +Lines: 27 +Xref: uutiset.elisa.fi comp.unix.programmer:181736 + +John Denver writes: +>Eric the Red wrote: +> +>>> There _IS_ a requirement that all reads and writes to regular files +>>> be atomic. There is also an ordering guarantee. Any implementation +>>> that doesn't provide both atomicity and ordering guarantees is broken. +>> +>> But where is it specified? +> +>The place where it is stated most explicitly is in XSH7 2.9.7 +>Thread Interactions with Regular File Operations: +> +> All of the following functions shall be atomic with respect to each +> other in the effects specified in POSIX.1-2008 when they operate on +> regular files or symbolic links: +> +> [List of functions that includes read() and write()] +> +> If two threads each call one of these functions, each call shall +> either see all of the specified effects of the other call, or none +> of them. +> + +And, for the purposes of this paragraph, the two threads need not be +part of the same process. + +jimbo diff --git a/testdata/testdir4/encrypted!2,S b/testdata/testdir4/encrypted!2,S new file mode 100644 index 0000000..b6470e7 --- /dev/null +++ b/testdata/testdir4/encrypted!2,S @@ -0,0 +1,57 @@ +Return-path: <> +Envelope-to: peter@example.com +Delivery-date: Fri, 11 May 2012 16:22:03 +0300 +Received: from localhost.example.com ([127.0.0.1] helo=borealis) + by borealis with esmtp (Exim 4.77) + id 1SSpnB-00038a-Ux + for djcb@localhost; Fri, 11 May 2012 16:21:58 +0300 +Delivered-To: peter@example.com +From: Brian +To: Peter +Subject: encrypted +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:21:42 +0300 +Message-ID: <877gwi97kp.fsf@example.com> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +hQQOA1T38TPQrHD6EA//YXkUB4Dy09ngCRyHWbXmV3XBjuKTr8xrak5ML1kwurav +gyagOHKLMU+5CKvObChiKtXhtgU0od7IC8o+ALlHevQ0XXcqNYA2KUfX8R7akq7d +Xx9mA6D8P7Y/P8juUCLBpfrCi2GC42DtvPZSUu3bL/ctUJ3InPHIfHibKF2HMm7/ +gUHAKY8VPJF39dLP8GLcfki6qFdeWbxgtzmuyzHfCBCLnDL0J9vpEQBpGDFMcc4v +cCbmMJaiPOmRb6U4WOuRVnuXuTztLiIn0jMslzOSFDcLTVBAsrC01r71O+XZKfN4 +mIfcpcWJYKM2NQW8Jwf+8Hr84uznBqs8uTTlrmppjkAHZGqGMjiQDxLhDVaCQzMy +O8PSV4xT6HPlKXOwV1OLc+vm0A0RAdSBctgZg40oFn4XdB1ur8edwAkLvc0hJKaz +gyTQiPaXm2Uh2cDeEx4xNgXmwCKasqc9jAlnDC2QwA33+pw3OqgZT5h1obn0fAeR +mgB+iW1503DIi/96p8HLZcr2EswLEH9ViHIEaFj/vlR5BaOncsLB0SsNV/MHRvym +Xg5GUjzPIiyBZ3KaR9OIBiZ5eXw+bSrPAo/CAs0Zwxag7W3CH//oK39Qo1GnkYpc +4IQxhx4IwkzqtCnripltV/kfpGu0yA/OdK8lOjkUqCwvL97o73utXIxm21Zd3mEP +/iLNrduZjMCq+goz1pDAQa9Dez6VjwRuRPTqeAac8Fx/nzrVzIoIEAt36hpuaH1l +KpbmHpKgsUWcrE5iYT0RRlRRtRF4PfJg8PUmP1hvw8TaEmNfT+0HgzcJB/gRsVdy +gTzkzUDzGZLhRcpmM5eW4BkuUmIO7625pM6Jd3HOGyfCGSXyEZGYYeVKzv8xbzYf +QM6YYKooRN9Ya2jdcWguW0sCSJO/RZ9eaORpTeOba2+Fp6w5L7lga+XM9GLfgref +Cf39XX1RsmRBsrJTw0z5COf4bT8G3/IfQP0QyKWIFITiFjGmpZhLsKQ3KT4vSe/d +gTY1xViVhkjvMFn3cgSOSrvktQpAhsXx0IRazN0T7pTU33a5K0SrZajY9ynFDIw9 +we7XYyVwZzYEXjGih5mTH1PhWYK5fZZEKKqaz5TyYv9SeWJ+8FrHeXUKD38SQEHM +qkpl9Iv17RF4Qy9uASWwRoobhKO+GykTaBSTyw8R8ctG/hfAlnaZxQ3TwNyHWyvU +9SVJsp27ulv/W9MLZtGpEMK0ckAR164Vyou1KOn200BqxbC2tJpegNeD2TP5ZtdY +HIcxkgKr0haYcDnVEf1ulSxv23pZWIexbgvVCG7dRL0eB+6O28f9CWehle10MDyM +0AYyw8Da2cu7PONMovqt4nayScyGTacFBp7c2KXR9DGZ0mcBwOjL/mGRKcVWN3MG +2auCrwn2KVWmKZI3Jp0T8KhfGBnFs9lUElpDTOiED1/2bKz6Yoc385QtWx99DFMZ +IWiH5wMxkWFpzjE+GHiJ09vSbTTL4JY9eu2n5nxQmtjYMBVxQm7S7qwH +=0Paa +-----END PGP MESSAGE----- +--=-=-=-- + diff --git a/testdata/testdir4/mail1 b/testdata/testdir4/mail1 new file mode 100644 index 0000000..a4e19c1 --- /dev/null +++ b/testdata/testdir4/mail1 @@ -0,0 +1,38 @@ +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "John Milton" +Subject: Fere libenter homines id quod volunt credunt +To: "Julius Caesar" +Message-id: <3BE9E6535E3029448670913581E7A1A20D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +x-label: Paradise losT +X-keywords: john, milton +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +OF Mans First Disobedience, and the Fruit +Of that Forbidden Tree, whose mortal tast +Brought Death into the World, and all our woe, +With loss of Eden, till one greater Man +Restore us, and regain the blissful Seat, [ 5 ] +Sing Heav'nly Muse,that on the secret top +Of Oreb, or of Sinai, didst inspire +That Shepherd, who first taught the chosen Seed, +In the Beginning how the Heav'ns and Earth +Rose out of Chaos: Or if Sion Hill [ 10 ] +Delight thee more, and Siloa's Brook that flow'd +Fast by the Oracle of God; I thence +Invoke thy aid to my adventrous Song, +That with no middle flight intends to soar +Above th' Aonian Mount, while it pursues [ 15 ] +Things unattempted yet in Prose or Rhime. +And chiefly Thou O Spirit, that dost prefer +Before all Temples th' upright heart and pure, +Instruct me, for Thou know'st; Thou from the first +Wast present, and with mighty wings outspread [ 20 ] +Dove-like satst brooding on the vast Abyss +And mad'st it pregnant: What in me is dark +Illumin, what is low raise and support; +That to the highth of this great Argument +I may assert Eternal Providence, [ 25 ] +And justifie the wayes of God to men. diff --git a/testdata/testdir4/mail5 b/testdata/testdir4/mail5 new file mode 100644 index 0000000..b12387a --- /dev/null +++ b/testdata/testdir4/mail5 @@ -0,0 +1,624 @@ +From: Sitting Bull +To: George Custer +Subject: pics for you +Mail-Reply-To: djcb@djcbsoftware.nl +User-Agent: Hunkpapa/2.15.9 (Almost Unreal) +Fcc: .sent +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: multipart/mixed; + boundary="Multipart_Sun_Oct_17_10:37:40_2010-1" + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: text/plain; charset=US-ASCII + +Dude! Here are some pics! + + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: image/jpeg +Content-Disposition: inline; filename="sittingbull.jpg" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/4QvoRXhpZgAASUkqAAgAAAAIABIBCQABAAAAAQAAABoBCQAB +AAAASAAAABsBCQABAAAASAAAACgBCQABAAAAAgAAADEBAgAOAAAAbgAAADIBAgAUAAAAfAAAABMC +CQABAAAAAQAAAGmHBAABAAAAkAAAAN4AAABndGh1bWIgMi4xMS4zADIwMTA6MTA6MTcgMTA6MzM6 +MzcABgAAkAcABAAAADAyMjEBkQcABAAAAAECAwAAoAcABAAAADAxMDABoAkAAQAAAAEAAAACoAkA +AQAAAMgAAAADoAkAAQAAAGsBAAAAAAAABgADAQMAAQAAAAYAAAAaAQkAAQAAAEgAAAAbAQkAAQAA +AEgAAAAoAQkAAQAAAAIAAAABAgQAAQAAACwBAAACAgQAAQAAALMKAAAAAAAA/9j/4AAQSkZJRgAB +AQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwc +KDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIy +MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCACAAEcDASIAAhEBAxEB/8QAHwAA +AQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIh +MUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpT +VFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5 +usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAA +AAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEI +FEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVm +Z2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK +0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDq77xdrX/CQ6laRXjRxQTF +ECovA/EUg8Sa6W/5CUuP9xP8K5yWQnxjrw9Lwj9BWjkgZHFAG6mu6yV51OXP+4n/AMTUq61rBB/4 +mU2f9xP/AImsJJTuAJFW0YDnfmgCTUPFGqWFq882p3G1eyqmT/47VfRfGGpawkgGp3CyIeg2cj1+ +7XK+O7zybCGMNjzHyR6gD/69ZvgG8zqU67vvRZH4EUAesJe6m/XVLv8ANf8A4mpf7Qvl/wCX+6b6 +uP8ACs+ObKdaeh3Hg9aANTw/4gurjxTLpU7tIv2cTKzHpgkH+n5UVheHGI+KWzJwdNP/AKFRQBzD +7f8AhMfEDEHH24j/AMdWrs0oCkDrVKJs+NfEsZ+79u/9kWrd5GqKTmgCstwwkyT0p5uzu61mOzbj +zSFn3DmgDB8ePLPe2MEQZyykhRzk9/5Va8D6Vd2Mz3d3CYxJHiPd16+la0hhMybkUvxhj1HWr5uM +uB0wMYoA3YJARjvV+DBPasC2lYsOuK3LVunWgCLQRj4sIPXTGP8A4/RSaF/yV2P30tv/AEOigDmY +QD408UE9ftw/9AFXpv3iFT9Kgs4t3jXxV6C+H/oAq5cxkMcCgDFltXVyMVVv7iGwtzNcNsQfiT+F +a8jbAxdgFAzmuTZZfEV81vG+xTyX/uIPT3P9aAIBr9vdNHcQI/lxk5DDBrfsLuK+jE0MqupPOOx9 +KzNY8L6fbaaYrdGXb3BOcnuT+FcpodzN4c8RRRyylrW4IDE9MdM/UUAes2wbOAK27PhRms6CJlwc +VrWowRkUAV9CP/F3YffSm/8AQ6Kfo+P+FuWp9dLf/wBDooAxrH/kd/Ff/X8P/Ra1evUOcgVW01Qf +G/izIz/py/8Aota2LqPK4xQBxniWc2mi3MxBGFA/Mgf1rmtEF/Z6HNqMNuzvPnY+7G1V6Hoe+T0r +qfGmnT3Xhm8WNWJVQ+B/skH+lUPBt3d3PhuzXyBM6xBY0YfKDnALewxmgDE1BfEDaPaXNzMRJPIQ ++TjCgDHb69u9ZGt2Us2lrdNDtMLAgq27Kng84Fd74qnaMwWB8qWRTnzUcfePGSOx4ziuf1kzT6S9 +tuRHlVUG5sDJOMA+lAHofh5/tvh3T7k4ZnhXcfcDB/UVuRQEdqzvDelPo/hywsJGDSRRjeR0yeTj +2ya3I8/3aAMXSU2/FmzJ/wCgbJ/6FRUunf8AJV7H/sGy/wDoQooAyNJXf448XYPS+X/0Wtb8ynyj +0rm/DIll8W+KDKQ0pvF3FehPlr0rvINMzbfN8rsc7upH0oA5ie3mktZSI1ICn5W43e1ec6ZrDwax +facIj9liUNtUcgE8j0IzXrHiqS20rQJbiadoyBsWQjc2T2HvXnvhbREuzeXTbvMlfILcsF6D6jFA +GJr+pWE1ymFkwFzhlwo+i1xevazLd3Fva2+UiQhh7kdPyr0jVfA8t0BeXNybe35UK2EJAJwST/QG +uS1Pw7HYalbKHUIxYxyDd8wHUnNAHsnhXVBrGhWkrBlmEYVww6sAATXQInA5rn/AOZtIa3mQHZI+ +xwfvAnJ6d8n9a6yazEKhlzgUAc1YAr8WbH302X/0IUU6xBPxYsSe2my/+hUUAV/Bdj5fi3xWJJDJ +JHeopY8bj5a5OK9AUArwARXEeFjjxh4xbub5f/RYrsIZgJhGTjcuQMGgDnfHiwnw1KJoVkUuB8yg +hfeuZ+HemTLpjx3OCZNzKUbPy54/Sut8Z263OlJE1wYgzkkjvgH86yfBb+XYWuIGiEithWzn9aAN +loTcO0ctuGjV9oMg5JGCSOOnp9K8/wDH1qH1iERrukRAqqB3Jzj9BXpsk6F+oyCuRjJ54rhNcg+3 +Ge5XiUSL5ZGc87sdPagDQ+HlvJHoAdo9h85mUY7dK7WSRCoB6HiuV8IiW10JYs7yszDJ7fN/k1tG +Rpb4xj7qpnj3Iwfx5oAwLMgfF+1UHI/suTH/AH3RTLJNnxltx2Olvj/vuigB3hgf8Vp4vH/T8v8A +6AK6aRWFk2CA2CPSua8M4/4T3xcp/wCftD/45XR6q32e1JjUySCRdqA4J3HH9aAKHiJTceH4mliK +r5e5lDfMpx2Iqp4eQR6Zp75Y4jX7xyfTn8q29djjbS/LMqxYGFdugNZWlskOh2pKgYj2AqO4OB/M +0AW7+NLQ3Fwi/O6hsk5yRwOO3WuS1qGJtNuvN3iNJkX5e+EIxn8f1re1e4ubq8jSOMiBArZJ/wBY +xOcfQcZ+tVNTsYh4dnjmG9PMJIP8XYUAQ20z2Hg6OeJGTYQzd+N3Le+RzXQ6TGwtjLLkuxAy3XAH +f8Saw9Mlt7vwsI4yZI9m07xtyM/y5rqodqxIFAIx1oA5iDj4w2ZHfS3/APQjRSw8/GGzx20x/wD0 +I0UAee+I/GV/4S+IXiAWlvFKJ7gM28njC+1Ubn4v6xclC1hbAq6vwzdjn+lXviB4X1O88b6lPBYX +EkUkgZXWJiDwPQVzH/CH61zjSbwj1EDf4UAbV78YdZvYPJbT7UA+7HP61HbfFXXLW3SFdOtSqZ67 +v8fesg+Ddbzn+yL3P/XBv8Kcvg3Xc5Oj3x/7YP8A4UAaY+KuvIQP7PtM5JXKt/jUF78Udcu7F7WS +ytEVv4grZHPB61VPg/Ws/wDIGvs9v3T/AOFMPg7XcHOk32P+uD/4UAWLb4l6vb2zQJZ2m1gP4WGC +FAz19q17f4va0sSobS04GB8rf41z3/CIayOuk3g/7d2/wqRfCWr8f8S27/78P/hQB33w78Q3fib4 +jR3l3HHG6WTxgR5xjOe/1oq78JvCmo6dq8+qXUBhhETQqJAVYsSDkA8496KAP//ZAP/bAEMABQME +BAQDBQQEBAUFBQYHDAgHBwcHDwsLCQwRDxISEQ8RERMWHBcTFBoVEREYIRgaHR0fHx8TFyIkIh4k +HB4fHv/bAEMBBQUFBwYHDggIDh4UERQeHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh4eHh4eHh4eHh4eHv/AABEIAWsAyAMBIgACEQEDEQH/xAAdAAABBAMBAQAAAAAAAAAAAAAH +AwQFBgACCAEJ/8QAVhAAAQMCBAIHAwYHCwoGAgMAAQIDEQAEBQYSITFBBwgTIlFhcRSBkRUjMqGx +0RYXQoKSssElJjNDRFJicpOi8CQ0NmNzo7PC0uEnNVNUVYMJRWR08f/EABQBAQAAAAAAAAAAAAAA +AAAAAAD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwDqjNudMEyw6lvE3HEqUnUS +kDSmeEkkCTB241Aq6X8mgSL5J/PT99DPrX3S232GwohPbp5/6v8A70PMsrSuwYKgkgtjiONB0Ovp +lykFQLhJ8+0Fefjoyh/6/wDfFBctsJWkpt0wRzSK9ft7cwVMNmf6IoDOOmjJ5/lB/SFejpmyeT/n +I/TFBdFqwACGGR6IFOWbRhZ3Ya349wUBgT0y5QP8qT+n/wBq3T0x5OPG8A/OFCy3tbZCRpt2j+YK +cotLMjvWjM/1BQEs9MWTR/LUn0VWfjiyb/72hki1sw5q9kaHL6ApZDFspQHsze3DuCgI6umLJqUg ++2Eg+AJ/ZSR6aMmj+UOfon7qozjTACUhlsCP5opo402AQWkeumgIK+mrKAA0uuK8e6R+ykj03ZU5 +JePx+6h6gaTGlPHmkUwzDjlrhDKVvtIWXVaUIjj4+6gKKOm7KZUQrtU+YCj/AMteHpuyvvpbeV7l +f9NAHEc3Ls8ed9iYZLTZIAI4+vpRAy1i7WKYS3dhtIDghQHIjiKC9npwy3O1rcH3K/6a1V034Ef4 +OyfPqlf/AE1WG1KKxAAHhFO0LgTCfhQTf47cGPCwuJ/qr/6a9R004as7YXcn0bX/ANNRLbqhsAOF +OWnVgEnh4UEm30wWbhhvBL5e3Jpz/ppwOlMLSC3lzElT4suD/kqHN642YT9tIu3T7h2UYHnQWW36 +S9YleXcQA8Qhf7U05b6SMPXiFpYGwfZfu19myl9XZ6leEkR9dVBy7XEFXLcUN+mPEn7VnCrplZQ6 +zclaFeCgAQfqoOqbO4L6VBxvsnURrRqConcbisqo9FOYE5mw1zGENdkLllpZQSCUkApO/wCbWUAY +63i9N0x3v5SkHy+aqj5QBXhFqRzbFWrrjrWm5a1ER7Unh/saqOSXP3Bst/4pNBbWhLY1nvCt1NyZ +mRWtuoKESCOdKjSOB99BiEjYHYU4ZQpKpTwpArTxpVt8g7RQPmlQkpP2UoFE8VkU0buEkK4A+NbN +rCzHDagfQCmN69TtsPjTRLoBA1b0oHFaZPwoH6IPODFJPwOG9asklJJ228a8egp9aBspcHhvQf6T +b55eZHWVu6m2kgISDsnaT76Kt46hlpx1WyUJKj7q59xzEFXV9dXC1krccK9/M0CybtYUTrMnj50R +uiPGLl69OHBTaWG2isgjdRkffQgFyoqO441eOiW4WM1W8D6aVBXpFAfGCVGQmnDZO07jmaYW74G0 +yKfNuJUBB50D0ISAIAiNyK3UUkQIpDtgUhAAIApNToAKQACedAotaZgCfOvQrUCAIpsVAma31kEE +bbUG60Dwn1oW9O6S3h2HqnYvK291E1SlEkzvQr6f1qRg1gVHc3B4+lAaeq2rXkFhcz3NPwcXWU06 +pzqnchtQ4CkBQjT/AKxVZQDXrpOEXLJO0XYj+xFVDI6lHAbEeLSfsq1dd2UPWyxMG7HHhPZCqtkX +/wAhw9Svo9gj7KC4Wx2MCKdt7ncTTaxbGmBz3p4hISNzHhQJrQSYnjXiUkQOYpZzupJJpFK5g7n3 +0ChQSkbxSluSFDfam5WACokj1NetPAqA29KCXb0HdQ9TSwQlSduVR6XBpJnetkXWgCTQSTQCRpMk +Uk8djzpsm9ExNaOP6pOqBQRWcblNtly/eI3SwoTPiIrnbEXAl7j4UaulrEHLbJ9z2ZEuKS2fQneg +Fevy62ASQYoHCHkqXsqrl0U3aWs52mpelLgU3v5jah20/C9zO/hVmyc8Bj+HjfvXCB9dB09bq0mD +zqQZUkAVDtriOdO7ZzSYO8UEoHBw3r0KB3mm6XUxERWdpO0UDhJBVvSulKh9kU2bUdzvvSzKu9tI +mgXShOjfaTQk6xZ/cOw4GLk7/m0XlqlGxEcCDzoP9Y1KhgNhI/lR/VNAWuqGpJyG3pBHdVPn84qs +rTqgqJyKyCI+bXHp2qqygH/XoQEW9i5M6r0CJ/1VVnJKP3Aw8QP4BvYf1RVh69K1lm0BMgXyQPL5 +moPJ50YDY+TCP1RQW+yRoaC44HhSygNXe99NbW7lCUKA48K3cdgbk8KDa5IAMcKj3XiklIMTSrj0 +zCppi8STwNAo5cEo4RXtq53pn1pke8NIUR5UoydWoA+lBMdrCdjSLr5G8yfCmJdWE6ZpNx2NiZoH +4uZjlThKitJ35TUOl4DntUlZOgoiaAe9Nl+tnDbS1QAULdlzxHgftoMLdm4VzAmKKvTsvsrywUqS +y8y42seBBBB9xoSWTa7hVwocGmyrh7qDZpUKFXzoptG77NdoFkwiXD7uFUvCMOu7+5SxbNLccO8A +TRd6Nsq3eB5mbVezq9l7ThsCTEUBcQvhtuKdWy0qVJ40wS4NqcNOTvA+NBKpMokRSiFKnx8opo07 +KAAYNOGwrko0DlB/JO005Y0gTNNW9pkzTm3SDJJoHIQhIlKZ50IesgFDLdgYgC74/mmjC03KaEXW +WBTlmykH/Ox+qqgJnVAV+8S3hMbOA78fnVVlZ1Oio5BbnkHI/tVVlAOOvSIbtlSN8QSD/Y0yyayh +3Ldg4BBNs37+6KedfERaWqo2+UE/8E1FdGdwF5Yw0kgzatmPzRQWdhnRClCTWPAEHeKkENodbSpO +xpBy3IJJjegjFwjh6U1eWSoxw50+uGQHPE86j7qEqiY9KBstYSe6B61gfLapB5QaaPOQs77V4twR +wNA8TcFyPKsdd25UyS6RBAivXXJSDQOm1jyinlq/CoSY22qDDh1AVI22mUqmNqCpdI+GXOZb9Ng1 +I9jZL+qNiVSAPiKiMhZQFixcHGGgkvJPaJJ4JBolsKaTdO9zdTcKV4CdqalLSrtevcQEkeVA7ytl +/CcOm7sLVKFugb+VSt06k3WgfSSO9HKss30NsagAkAd0GminCXSr8onegfNKSDO/nJp22tCSDUYF +xEJpVDkkwDPrQTTCoGxBmnjDp4KG9QVu4rVxjepS3WY1KVPpQSbSiT508tBvBImo9hJICt96f2yS +TwIigkWht6UHes8ojK9mJE+2Dj/UVRhaEpn7aDnWhH71LP8A/uj9VVATupmZyCiePzn/ABVVledT +OfxfszG6XYj/AGyqygHfX3A+TrPmTiKZ3/1JqvdFX+h+EqPO2TU91+NrK1Mf/sER/YmoLopSpWR8 +II/9sPtNARbNSdAKdzHCtnY0Eq4zypnbFaEAqImnRUFt6p93OgjbyEEn6qhrxQUo+FTGIp7m23rU +HeGKBi6E6orRcRtWyiSoxWp4ERQeoAI3NeLSnnW7Q2341s5skcOMUCSEpMk08to0gA/Gmu4HCvW1 +qoPMQWtOJIDR/I3HiKyycacfU4o86Y4qpxBcutiUphIpLLy1LGtSN1RsTwoLS0SQpx3uoG4HlSHa +gq1JMSa9ullVulJAA5+dN0wYiKCQbcJ+lNLoVzmmjKoEmndudSoVAHpQOrbVq3mIqWtYME7Co9oE +xHAeFPWVqCQY250EsyuBsSPWn7C1CI4+tRdq4F7gjzFSduoCBHoKCRYUYAG1CHrQJjKFqqB/nqf1 +FUWbck96NO3jNCXrPmMm23Ha9R+oqgJPUvM9H6PLtR/vTWVt1MkoT0eMaQAVpdUYPPtSPurKAb9f +ZQNkwnf/AMwR/wAA1GdD7QOQMHUeduPtNSfX3H7n2y4mcSSPT5g1G9D69XR9gxCeFvHHzNBczGkR +tWqXIBjlW4HdO0U1XqQsGOe9ApcAKTHEGoe7tyJqWWSBv61roQ6nvCgrD7cEyPqpNCYn0qavrMhW +w5U1NsYAAoGLAEmaUcAinItSZ8R4VjjJ0Qo+tBHK+NeoI0kVXs3ZptMIS42wk3FwB9EGAPU0KMWz +zma6uCtm67BoHZLSQEx6nc0BdzRfBq0UhKu8BwFI4BchttK1EyrlPCqjhmJOXlrbquHUPOrbBJJ5 +8+FaXC32NTguLi2MmCYU2fLjPxoCel0vDUFFQPCnSAQaH+Vc625fRhuKpbtX+CHAruL+6iC0QtKV +JUFA8CKByyFE8KeI2CTzpG2E8h8acpQSRCRH20DxlRMGPqp2zOkJB2prZtlRIg/Gn7bSUwNW9A+t +BAO4B8Kk7UeJmo20SUkmeHlUnbK2G1A+aiZjfxoS9Z8k5Ltwrleo4f1VUV0d6IG/OhR1mkg5HZkz +/lqP1VUBI6l5H4vmRJOzvHl86aykOpOf3iEb/Sdj+0rKCgdfUj5NtxzGJI9/zBqP6Gkj8XuDDn2H +/Malev6lHybakQD8oNg7f6hVMehVIV0dYMqJhj/mNBcm29Q24U3u2gDtUohvS2dtzTO6H+IoI93d +PmBTcq0xBmnDoM7b7U1KZUQCBvQLKcQtuFRSCwAmBFIvBxJ2MGaVAUUhQIngaBNUg7kVV81YoUhV +rbuweCiOPpUxmC9Rh9it5Su+dkDmTQ9Nz2q1PrO5JCd5JNAxxyztBaOOXyktsNp1rUefqaH7dpc5 +hvlDCLJabVJgOLG5H2D0q7YhhV5nDFrfBLYrTbA9reOAwAkHYep/YKL+X8qWGHYe1aWjCUBtMDQn +c+cmgDtvlq7scEQ2grTdtnVqKdhVXx3EscYTpumoIkdogbKHmDXRWJ2NotlxpbjRUOYVJ9N6G+bc +Mt3EdghaFx9IAUAXfxB25bIe0haOHd+qiR0QZ1dRdIwXFXdbayEsOqP0T/NNU/MuXH7Qret0KKUn +dIE/CoCxUW3UnfcyCOINB2BaiRI8Nop+0gxP21R+iPHF45gCUPq1XVsezd347bGr8wlQERQLWp0r +Gw41IoAKp2k0xQkg/Qnwp9bSYJSZFA7YSfjUhbN6uMAimTBIUCakbZW44UD1hkJAoUdZ9pKMgNqH +H25H6qqLTR7oPOhZ1ngD0dSOIvGz9SqC4dSZJGQyeRU7H6YrK36lBH4vk7flvQfzxWUFE6/oBwm1 +Vz+UWh7+wVTboPSR0cYKkCf8nk/pGpHr/o/e9aLjf5SbH+4XTPoOUk9G2CyCT7PwA/pGgvaR3eAN +M71E8qkkJBRsDvTa7QNBFBCuoHGKaqbGuZEU/udIERvTbUmSNO9A0ukjSTG9Nm1BC5PDnUg8pIHC +ozEn0tMLcCeCSfqoB9nrElXOLi2bJ0IOkeE86rVzcBDKnAoBLRgCNyfL/HjXmO4koPaoPaOEmf5s +86aYHbPY7mmywdk6mmyHbjwKZkz60Bl6NcGRhWX0XDrWm6vPnnIHeAPAe4VZx2iEFSVrKYgpI0/X +UJeYuzhrae0Qt9SANmwdIH7ajh0j4WrW204dYEEE8NqCXW6+7dP2yUMNnTKdZJ24cao2ZGCVO9ol +tzcmUnh/j7qXTm3tbC8xdcKQnsWRJ7pIUoq+qKqeMZ3YUp9xxxsJcUCQngOJ2+NAmptDxKHEqSCN +vGhnnCxOHY8sISEpe76YECecfb76J2GXDV8wLj2a4DDhlDikEA+h8Kr/AEk4St7Chdtd4sHUI4xz +Hw391At0JY38mZntmnHdLV38y4nlP5J/x4102ywVAEbiuLMDfcYeauGjpUlQUDHMGu1MpXPyll+x +vkbh5hKifOKBZtuCRoiKWbQoGYpx2KkmYO/lSiGjI2EeYoMaQSSDTplJA2rxtJG1OmUgD/tQKM6h +50MOsv3ujlwgzF02ftopJMAAxt5ULeslKujq5I/9w2frNBb+pM6Dkfsuep8n9NH31lJ9SUfvOBgb +F/cf10VlBVuv7q/B61g7fKTW3/0Lpn0IjT0d4IN/82HPzNO+v8Vfg/apkR8osn/crpr0HEno5wUE +cLeJnzNARGSCmCD8ab3IBkAcqctjbhwrV1oqTMAUEBdpABnemRAHvqYu7WASQDNMOw7xAKfHjQMn +YjvDaq/m64RaYFevcNLSj9VWp+3IEyKo3S8U2+RcSXIkthI9SQKAKXV4u5ue21BLY3JngImrb0S2 +TrmCXuYVuKb9pfUkr0nZpA4Ty3maHDSXsQLOHW/8PdOJZbHLvH7AAfjXUOW8rWWGZbssMTZtutMt +BGpbQMniVRE7mTQDzMfSJctYXcpwbLq30MoSFP3KinXO3cQBJ9SRQpu/lvE8WS+jCQXXVAyylaUu +T68TXQubsNNpZK1YwLVsDV3mSSPSFCoPJeTH3rv5dvO2UiYtu1RpKxH0o4xvzNBHXuX0WnRpcGzQ +supWlSyqZMjccfGN6FGG2+Iv4u5cW+ALxNLSoaZUfmwoc1Abq9Jrq3GsPtrbI12gcFNHaI3oJW9q +7Z9qbEpSpapg+NA2ezHnJ4rZxLCrZLVshHYWzdutBXPECCoAjzqRtlKxNpxp22U0tSTLS4kCPXen ++XLW5xR2EN3iXE/SCmwUj3wD9tTSsHLPdUtK1cYiaDnq1bXa4jcWLuy23FAe7b7q616v1yb/ACFa +JMyySg78INc19IViLHNAuG0hPa94jzTxo79Vq/DtjiWHDcJWHk+iuXxFAXrls7xO1etJkCCaevNy +NXOvGGxG6QDQaIbM8AactohMRWyWyD605aSInwoGwTIgCD40MOsa3/4cXh8Hm/1qLSkhMqNDHrHN +T0Y38Dg43+sKCd6k4H4EHxDj36yKyvOpTIyZpJPF79dFZQVLr/T+Dtt4DEmf+Cuk+g1uejfAzEk2 +wn4ml+v8EjK9sYknE2Y8vmXKT6BQ4OjTAyT/ACfh+caAiBoDYCaxxOlAGnjSralQJg7V4+sdmBtQ +Rd2gaZM7nwqLdShCzE1NXS2wmJmot5CSdSQregY3KthE+kUNOnjtF5Hf7PVHatlfoFCim+hOkwDI +86p3SRhyr/Kt/apaKipklIHGRuKDmjKV81ZZuwi6cWSlNzpE8BtE/EiuqLbMPtFi2WVDVA2rjXEy +5bX1skApW2AQPA6ia6awJQNvavpSZWgKI5EEUBLwmwbxFSHMQYZd098FSQQD761GMWuK45eYPhYT +cKsUJNw4ncIKphI84BJqs5hzKcPy4+q0lVypISNIkk8gPEkmKnsuZCvMHyEi3sMQFnmC6V7Re3Rb +16nFcUnyTwHp50Epmxm3ssnrYeUe0uEHSIoIYg0cKS9cOW5Uw1ClqHEJ5n1q59JWKYhh7tthN0+u +7et2NetKY1pHExyoft4lfYnir1u6sPWL+mUqT9HxE+HxoLhaXS2sND9k5rbdROpJ2INa212OzK1L +nn3qhMtMPYU67hThKmEElon+YeHw4e6nFxPblQBKPyY/x5UAt6VLwPZoba3DYbn3knf6qLXVGbcV +e4u8QezS02kEjmSTFDHNuXcRxnNDKrG3U4SnRJMSZ/711B0I5OcyhktuxvA37a4suPqRuJ5CecCg +ujoEQfCtmOI2rZ0d2vEpEbcKBy2AVcvfS6UgbDnTNJPnSrRK5UmRQKqC9ESDvyoY9Y4q/FlfiQfn +G/1hRPWogaYihh1joPRhfEj+Mb/WFBMdSon8DgDyL8emtFZWvUnCRlFYSRIL0+utP7IrKCr9f3/R +q2PH90mR/uV170EKKejLAxAP+TbfpGvOv+CMsWpPPEmY/sV1r0Bk/izwPcx7Od/zjQEhlRUNk+6l +XEymY5UmyDp4x50uggpHemgj3mZSSRA8aadmEqAVFS9yfmzHhUU8UzvyNB4tpBG4NMLy2YeZWFN8 +QeNPA4QdMkT9VIvOHSUhQ3oORulXJl7a5tvbq2tyq3U4FIQBvBAO310bMBsQcKw+4ZIIS2kKSR5V +KZ9sPaWm1WqW37i2lSxH0gBBFUTLeamLW+Tg97cBt9StSEcAB/N9aAws4LgLCbTEbrSi3aWLo61d +1Kk7ifQ7+6vXeke2uLdCsvYLf46tThSk2zKuzBH85wjSBVeTeu3rCMPNom/tnFauyUdKfzvETyqb +fubpFs2wthtCUCENsSAPACKCk50xPMvt72L4nkt4Xa2SygtuIUkIIjeFHkfKh5huMWdk+W73Bb2y +UDu6WSpER4iYq45rwS9duV3Nz7Ssap+ceJ5cAJquM9o2QEIJ7wkKPCgnsHxCzxa1F0w6haSkplJ5 +UjdH5tdwe4gCRJj/ABzpo0lq3X2rbKGVLBBKe7J9OdUnpQzULWx+R7R3/KHk6VkH6CefvNBMdEmc +13PSMm1u3W12C3VBgFIlJ8Z84+uuvLIzaoKtyRXzpy7fXFjcpWypTa5ACk7Eedd99HmJt4zlHDbx +D6XlKYSFrTwKgN/roJ1xHdFY2J3mt3EmIrUCO7FAqhAVsRtThIAiCD5UigGCZO1bpSqNiaBR3SpH +CCKFvWNI/Fne7TLjY/vUTl6tJoXdY0KPRnfGP41sf3qCb6lP+h7h0gDW8J/PTWVp1Ilzk59E8HXf +tRWUFX6/5P4NWoKhHyizt/8AS5SnV+bDnRpgaU7xb8PzjSH/AOQNUZfsk+OINH/dOUr1eFhHRlgg +3nsJHl3lUBONvGxTBitUthPGZpVLx3kztz51p2xJ2igTuWu6ajLhvjvUs8XltEttqWI3IrZGFJWl +K7l4hSk6uz578KCtLQrXEz4RUknAHT2a1uaFkayI29KnWcIsrZ/tN1rSJgqmo1u5vLpt1p1pTUPE +srk7jjvQBvH04hhuarxzELlaGn0Q2hHBJ3gTXOefrwt5oadUl1p9h4HQvYxqkGuoOmF+1U0bJ25Z +aeWsKMqEq8QJrm7F8PcxfPTLKy12LSzD6mtaFJBkBQHHwoDBaYld4c004SpbRAKVDiAeVWFrPNi1 +aAugrlOgEHcbc6jsv263sBZtbhse0sICXE6eGw5elRSsJtA4/rag8Y+6gaZqzheXlyvsSWrdHdSh +KREcKgm8bt0IlxYTGxkQaZZvtW14u0D80yGwVRtJBqCxRy2ClOaU7bJkzvQSmI5mff2YgNjYeJNC +/MTy38auHnFd7VzNWW9uDDVtbgLed2QmOE8SagMesfZr0NKJU5pBVvxJoNMLt3HvyoI70zw3/wC9 +dP8AQRnVzD7K3wm5uGlWSTpSeaT4ek0CMm4GL68asw65LpCNCB4kcffRz6JMAbX0iXOEBxsJsWUO +IhI0qBH+PfQdDsrQ4RpcSraYml9AUkq2EVDW2GO2OKe0hsr1JIUUq2I9KmEPs6ezSpJWRtNAow2C +jma2TqA7vCvGFOCQpPpW5PmRHGg8UCUgn30LusgI6Mr8CZ7Rs/3hRWOyBznwoXdZRMdGV/wMuNfr +UDvqQz+CNwTw7Z0Ae9FZXvUiH70LiOAedjb/AGdZQC/riY9cZgybaXdx2QWnEGUKDaFIAUGV6tlC +eYq1dX1BPRrgRHO3/wCY1WuubhNnhOUrC1sX3LhtF61LjiYUo9k5JMVaur4kDoxwLf8Ak5P95VAT +A0rmB8a2w+1L9ylomATvHhSrTetMJBJPKp7B8P7BsFMdqrdRPKg8xK0Qzh6bdlYbkiSBvE71AnCc +TxHGRcqAtrdteyie84ANtuQq6KZSmCoaz4mtVpII22oIe2wq2t2tMOOL3lajJ3qDzc37Jhb2lSkF +YgL/AJm3GrhpUkn3VSOljMWDZawBy5xd4gOgpbQlGpSj5Cg59zRaP67w35beeZAUwp8gl1JO59QK +p/RVgS2XL/GL5am+3LyWg2jVI4QBB47/AAqbxvNb2a1vMYQ+i11KSEQ0QVeEq4/DzqwdGjLlvlkL +unHEPquH9bkAqRGqVAcyDvHlQXHEMETbu219aJKA+yhtxB5EDYn7KrGYLY21yEOoLayCFBSYNF+8 +w4sXFu+EqubNTRVp4qVpbBA8NzJk1TukZeMu4fdN4ZgaMTcaLIDIZDim9X0zMgwPf6UHP2e03Tt/ +bN2rS3HFpACW06lKPkBxppYdGmasQR7RiCRhtvpK4cV84QP6PL311ecm2jFs0xgybSxu1sIcDxZk +lJ4jYgmD9tVvFsoWiQ45mDHVvW8QvfsEQDChtufpJI35Gg57Tl/C7ZDjdtcsWz7SE9s65K3EhXPh +z4eFVvNtrg15b9ph63W32nEMtoKCS8Oa1K4DyFdE57s8k4060zqQbi2SG0sNgw547jYgcffQr6RL +Sywu6ZFtbN2zCXO0LaR/CECdz/jlQVnLWH3RxRoWoLEaVLUtUwf5225H30XeifGlYJnO0tn0ds3f +/MG6UgJgkyD6HhVUyW3b3l+wEpSxcusjSDJC522+HD0qy4Thq8VzrY2QcKmRqaeCO4oECAoBQmAR +QdTWTYCAVLCieFe3WFWV4pKnWh2iFSlSTBFQfR85eHD3MOxLvXlkoNqWf4xP5KvfVwZbIIOxmgi1 +4elgSkrIPDypqtlQE1ZnW0qbg7yKjLxlTKVAJGmOJoItepKR3gDFC/rJKJ6Mr2AdJcb3P9aig4VE +bDj4UMOsekp6ML2SILrUfpUEj1JmyjJrytiFOOnj5oH7KyvepPIyfcTzccP95IrKAb9dN9p7J1gb +a+urtkXjPefIKpDTgIMbE93iKtHV1LKujjL5fC+z9mIMeSlVS+tZh2L2HRrZt47Zi1vTiTQKO11w +OzcgzJnh4mip1YcNb/FTgDrqQrVbE7jh31UBOsLcdhrbZ0JiEDnUvbM9miIk86SsdK3VqaVqbTtE +bTzp4J5UGq0kxArRxsngDTkkgbcBWizwmRQNXGjvFBHrLZYxXHUYabJhbzDetC0gx3jEb8uHOjq4 +ElMzNUHpcfScEFmq5RbofVC1qXpgDfY+M0HI/YYtl1a3bmyt7C3LnZ26zHarIMaoEmOfhRKyM8nE +7Fm4tihLaFKWHNMBKtCiVkb89yKHXSo7a2+IlouqKWkKTDjsjURsB5wZnzq+9ByG2cCtmlKI1rB4 +mSSOBI5GfhQG51SMLwhd9d3wWtNqhaU6YTAG+mPEmn2AN2tvZm9cCW37sh1ZPiYAHwgU6Yw9rEcv +27ToAOgBJT4D9hpZGHNuhDK4T2YiOQoEsVtnlLtvZ7QOuIUdLmqNCSNxHP0oZ490dXl5jryPlJ9F +m6kOOslZWFKPE7kxwG1EZnGGnMSvLNKoatEEvOzEE8BNKNOYa7iSk2zrbtwWwpxSSVGOUnnQU+wy +Rg+HIQpNqhSkc1JGxoE9OeErv82Iw9huO9Ko2CUwkn7a6rv7VTiDpMeBP21zn0j26lZuvLy3e1Ia +CVSonhI24eCaCjWtk0xcW5xC8RbtWoCmm0nvEJMgGOG9WHo4dxRzOthdXGHutoQ+oF7V3QF8IJ3P +ED31EvM/InbEqZccvHNRURqCWwIjf3mpfJ79xdZywtCriXC+lxSNR7yAdyYMAbbCg6WbZXa4na4m +2SEqhm4HLSfon3GPjVub2G1QVtbC6wxbau6FoInwqTwl0uWqAsy433HPUUEnJg7CKbvoC0qB4U4k +cfqpFwlSjHKgg7pstrIKNPhAoSdZhSm+jW4kDvXLSfrP3Uar5jtGpmKB3WkXp6O3GuZu2p+ugn+p +aQrJTp0gaXHUkxx7yT+2sr3qWQMjL8S69z/pJrKCqdfhR/B21bIgG8ZIMcfm3Pvq79WMA9DeXdWw +9nV8A4qqf19Qn8F7NYBKxetD3FDn3Vaeq68X+iDA0L/ikKSNuI1E/toDCNKW+7tXjbknTMKB3pNR +BRusSRwpG1bQlZdTqle5M0D8pIRIMzWi0k7kVs26VARuOHpWy9kydtqBvyAoR9PDdy6W0NBOlLJV +KuAMnf3UXlQTx40JOmrHLe1vfYLllLluGPnSeRPn8KAA4sxhGY0IwXEbdAcaKSxfJRpVpJAjVzTx +MHwFGPLGXLXA+yw6zh1lgobSswZKYG9B3GUWzCEXdmXHGFPoShKR/Bq8FetHnJTbjlip1/6Yuz9L +YjvJP7aC9ZYYPyUyoOiQkApSQUp8hG23Davc2JxC3wi7fwlkvXvZHs0AgFR8idp8JpbKaAjCGwmS +EqKQeaoP0jtxPH31LPJ7VlSRsY29aClZTwFdnkq3axBhab19IuL8FYUpxziQo8Dvy4VL4Szh7XaN +WduGlIjVCCBvy4fZUi2T2imjsJmDSVuy6h59bjoUCrupH5I8KBtjTybTCLl8qCSlohJP84jb6656 +zk0FX+IS0pbhW2lKUnc7EzRw6QHpwL2clZ7d1KShIOpQG8SOA2Ek+nOg3iuFrxBV+Ge0S4HHXAAd ++4Dtz5GgodlhTuLOXV9blt+3t1qTcIKtK2FA8DOxnl76neizALh3NTd4ppRJCVIIJ+jJ+HCvcq4h +8nZdRamwLpecU4tLg1BaCd9W3Ab8TPCp/oVxC3s80Iw95d2lVwSlhLoEJgcoHODQdA2Dei1QmDJG +9LJZ7N4vN7H8pP8AOrGNSedOAQU7+NB6l1KyIO8SKb3L4QstEwTBn1P214NFvreIJK+fgPCoyzWi +/eXfPhaQh5SG0qVACdoMefH30Ek+4lCi2dwQTx50Detesfi6J4TeND6lUYbgqf7VsK0KSe799BDr +VOLPR8pCjwvWgY9FUFr6k8fgSuP/AFHj/eR91ZXvUm/0GV/Xe/XTWUFc695P4NtJO6ResHj/AKty +rb1YRo6IMvAAd5lZ9/aKqqdexKjgCIE/5Vbn+45Vs6ti9PRLl0Dj7Or/AIi6AqO6QtC1qjkBHE1o +HCEKCdlQSJ4DwmtL3UGwtKgkpM1s22OzCn1pJMcOFBtgq1rtRJKlT3jBAJ8p4ipIokAz7qq+FYnj +D+OutrtbVOFto2eQ4dQVJATEQdoPvqypdA3kEcaDCxJ5jehL0k4JbX+aF2D4Kn7tslsaZ2jaD4yf +qovJeBA9aGGbcYUxmS4XcLW2GwopMDToHL12nagDeMWrOQ8Ut38Te+bfeShTDYSrWUwZUOW870UM +MxBD2EX17bgFKrp4twdjCEkfZQn6VMHxDMuZrB/CXbd3tnA0u2dX88gkzqG26efjtRiwWy7DLhY0 +gr9ofQTp3nSdx+jQXTLLxUxdJVGpL6jtMQrcRPkR75qQQ+kKUkq3mq1ktSoeSlBQkttqEj6RKASf +MST7wamXwWX+2EQdlDwoFApS7wmBoB2ptbG6TdvOOvsG3OzTSEFOnfz4+tN7fEW3seXZNmS0NawP +yR51o9d3Vzj6mBZhq3aSfnCoFThkbwOAoIHON2HsXtbALWgBQVISd1KJgTwiEmR6VV8rgqexG9AB +0WjzgJG0mpa5Wm4zViN8hQcRaoXJ0gAdmgAAEHeFFcnjO3KkujlTSbK+uX/4MMLCpH5Ox4e80Alv +7O89rtLK0tlXK1EgISqEK27yp2HAbDyqU6MsLXhucEqWyHHGnNTbhSJIVPCCY9Kl8CtsYvbxzFuz +bu7dLmi3t1PFGpO41pjhyEceNPMpl+zzkwzeKLr2IK+ilOlCAlJIgevPnQGFNwrsCofSArXA7x29 +Sokd1KiJ9K2QzpSNUwRxpLB22MPs3xqVHaqWeZMkmBQa4u+3fONWdqVrLdylL+gwWiBqTPiOHuNM +ct3D14L9zswi1RcuIb1cVadp9Nqe4241h6HMQbMLQytao4KhOxPjwpnktl1rKNqHge2cRrcP9JW5 ++2gmrRsItgTBJ8KA3WzRoyUFJ4KvG5+CqP6GyGUgAcKBnWzQT0epIH8ub+xVBYepbIyQ2AQR8/Pl +84Kyk+pPq/A10HcBbo9O8jasoILr1ScBRBgh63j4O1YerOo/ilwHURs0sbf7VdV/r0j9wkqn8u3+ +1yrB1Z5X0TYFwjslgf2q6Aq3X+bSPCeNIYchNywQ5JTBETtTi4SVWy4HLamuEhSElJM78qB1CGG2 +7VpCUpnvnkkePrUQ/cXjOYmbFAUbd4FSf6McR9dSmINLUJCCptB1KTzX4D0mqhmPH3MvZgZvbxtR +sy+1bKITMLcCt/iAKAjhsJQIiYoW9Ilp+6V44yg3DhQIQ2qTJ2I08440TWHw/bJdSkiUzvQ2xxdy +xib7rDalvF8nYctXCaAMYfjWJp6Q8Js8SY1D2xBCwkoIIOwE8eKZo34OV+zW0pK+1vS4QFcAttdD +XGMYwu66SLG4xKzadvWLhDbLikFKgeB9QJPwotYez2YwxI4FDK9/zx+2gTynCb20WVhRfsQAmBKA +hRmY33Kvqqw4mlCLRxxwEpSJgbk1XsHhl7D5b0Bu5fZU6Y4EylBnfcmdvDzq4ONpXG21AN8i4ZmH +D845gfxLsBZ3LiV2znaa3FbniD9EAbRw22qw2bTFniV865dXLj4BUpK1Hs+E90UviDqm8UJgCFAH +zBH3iorPVwm1wdy5ceLZdSGUKClJAUo7GR/jhQVxntG8v4ze3LTSHnWtB0DYFaiYnme9x58aQwNt +bWT8XdSrQPZwkq8CZn6op1i/zOT20JVPbuBSTxkAEj7BS+VOweyhiDlwvTbuFxKlEcEgQf20AJuF +Y1eW9vb2jtxpYK3FqZdG0ERPDxiiV0WYWu8xNGP3ilpdb7gSsGCqIkVBXgwDCMFQ40t5Srx5QZd1 +hSVRAkDkB4bVa+iS+YcuHMMR2kBHtACySdzvx5cKAj3t8pi1WUtqWoJkAVC/L9naWKbh1Sbi8WUM +9i2QopWQYkDcDjvUvi94zZYY/dOphttsqJAnYChDlN22ds14xiS/ZXrm+cuWme00rUClPZpUBvJA +n86gvrz1zieBui4T2NzeOIY7IHZvUdwPQA1dLVkW7DbKYAQkCqlhlstzMOCWOgDskru3gTMHTpAJ +9VH4VeFp+cMAbeNBouNEBRG24oJdbFH/AIcpM8b5refJVG90d0bbmgp1sEk9Hze8RfN7fmqoJTqU +6fwKfE97tndp80VlJdSrbLFyJnvu/aisoIrrzpP4OoVy7S3+1ypjqvuT0SYIkk7B0f71dRnXlSDl +hKo4Lt/f3nKfdV0a+inByPyQ6D69qugM6BLShHEUyw7a4dQDMHantuklBHKmjbfY3ijBAO9BviT7 +bPdeMJcQQY/Z9dQV6S4n2e+CXgp5tbIUmSUpgyfMGneeGFuYCt5DikKZUlyQJ2Bk+u01st1tWYbS +zCNZuGlKkD+DSIG/hMigstuEhlO0gjaqRmppJvXvY3GW7pSiYdMJkDjPjV8Q2EtgDkIqi52tmlG4 +L3dE90zxJHDyoAnjDdwzm21uMX7uKXVwA2yIKUJTxWFDZRO3186NzjqWbXCVE8EIT8Ck/fQlunm3 +834azcWVtqt3VJQtLkqEjf7d/Si9i7KU4HaPJ4MupmPAgj7qBtdtqRbX51hPs16h8o2GuTATJ9QR +5gVamHSS2VRCk+FRl9bj2m7HY9qLi2JCAY1KA2G+wM86c2F12+CW7y161pQNStpkcZjnQNcetwhZ +dA7y1JHHjVTz5cLdcw2wYQHpd1OTuECISvbeZkDx34xVhz1iAscKNyYKUd4jVHI0PsOujfZgaxC5 +uVIbZSFFtPNOkOBRj1IA8vSgks4OMpNtZtAFthWohJHArA4egVTjCbFf4txag6XHWCtfA7rOoj6z +UFfF66XcXUavablQSTP0Q2qOJ/pD3ir4y2G8Hbti1rBbCVDgIAoOfsaxFOG481hC3LZ21QrtWm1t +hayrgrSeRIMRwgUR+ibCWkdpj3cC32g2hCTPZpHI+e31VRs2ZYtHLxePKJUwpxTSWUcdYPDUOCSI +38iKu/RG4ub+xDehhhSVISeI1DfhtxFARnG27izW28gKQRBBHEUEsFwkXnTJcYeo6rSwWl9CI4bC +J8d/so3gAtQKpOG4a3adIuL4i0nvOWaJ24qkxQWzKzYdxbEsVIOnULdonwTx+s/VU8rvKK5nwApD +CrX2TDmrfYECVkCJJ3P106aSJ8qBEqIAEDhQZ61xB6PmduF83+qujY62nwoJ9a8j8XzQif8ALkfq +roH3UsP72LoT/GO7e9FZWvUtIGWn0wZK3if0kVlAy68IJyxO8Tb/AK6/vpfqquBPRfhQJjvuiDz+ +cVSXXg3yuE+HYH++uvOqopA6M7DWB3XHY/TNAemBKCBSdy2lDqSY350lZ3KdxIn7a0xJ0ltCkmCn +eg0x1CXMJfbIiUECee1UToGzLe461jFtibRVcWF4WUPkfwiOQnxEVaMRxGbRYWZEGo7oktLS0yym +5YA1XTzjyz4kqNBeytU+NVvMFlbYip5t5TqVEBIUhUaTyPhU4t4p4c6qGbHL9DinLI/OIlQSTHLj +QDfOeCsYNmBDlpZEvKR2i7kypSjqg78Ad55UUFLL2WlIUkmWwRtPDehPjeM3t04W7nuBtxsFA2JK +l/ZANF7C1JVh7bSu8kpjc0Gl1iLbVrh12tQRKkoJJ2hXdH1kVtgbgFvdWyne1LbihqkHUPHbn4+d +QePYbcYng7Fqw5p7J4FSvNCpA+IFLNXTeFIxS+dSUKTal5QJ2UUpMke/9njQD7NmNX+dM7KyvhyU +2+HWw7R+4c37VKFaVCBwEggTE0/dbfQWbewabsy6pWs9nJDadjII2Jnlt4VZzY2eH4cvFbawtkYl +fNN+0PBAG8TJ99Nra0Q7jTSyhLa3GktrK3ZEDcpTzJkn4cqDW8swi5wu23AbaU85pHFS1AD6pq2L +HzACRAAqKdQh7FiscilA9E//AO1KuL1IMcAKATZrFthgdwnEb1NjhhJdU+133DqV3U6dJj135VNZ +CxrK7PZ4Xg945erMBb3YmSTJ75gR68KpvSFdtXV3ibz60e0W1ou5LSRsEAqS2k+oMnwIqMyLauYM +3heI3lk9bKfulWt20pWkgLIgn3EEUHQEd07kCq1lwPO5qxRTgGhCkBJ8QBP2mpe2eXZdnaP6ltaQ +lp8qmT/NV4Hz5+tQVveLtMcu9AkuOoB9KAgNqUobEA1oHFa9JFJ27mod0+tK7SFzvzoN1lWmZ4UE ++taf3gsTzvkAforo0uujSQN6B/WqcKskWyRzvk/qroJfqYT+DTuw2U9+sisrfqZD97D3kp2fXUis +oGHXc3y3Eb6WD/vF1TOr7mvCMJyFbWt3i1nbPJdc1IceCSJVI2NXTrrtzgIUebbH1OL++uNnEkHa +RvQd42PSFlkAFWP4bI//AJKPvp09n/La298cw0+l0j764C7yTxPrXhKualUHcGN51wJVo6pvGcOJ +0HhcI++k+jDOGCYZlJi1uscw8LC1qANygEBSiY4+dcSHUocTvXg17gK4UH0Jts/ZXUolzMOGJA8b +pG/11Vcx59y6rGJbx2xcbKIMXSAkHj41xBqXEaj8a1UTO5NB1FjucsuqddGH3tj3HWe0V24AUe/M +c1RIPGN6I+HdImVm7VtCswYYCEj+Up++uFIJ8ayFfzj4bUHdFp0j5TQ++hWYcNSgr1g+0p58ai84 +dIeUVWi1s47h90lbKmXbcXAhaVbevjw8fSuLAFR9I153juSZoO2nek3J5Nuk5isAlpIIT2oIGxHv +2Net9KOUWFBacdwwlcqWQ6JnhwiuJAFExJisg8N6Dta16UMn9sFuZhsATqO7nifup4elTJn5OZLD ++1rhzSrzrIJTtsaDoPpMz1gb+NG7sL21uBd4Y7au6FA7laon0mansdz9lrEsh2S14tZ+2pQytbYW +NWtIAPv2rl7SRsTNbBJ8aDtW36UMn3Fg2HcespU2nUC5wMVBPdIWWk4s1+7to4nWklwrHAHnXJAC +toJrfccKDuWz6UsohO+YLEf/AGU5b6UMnkE/hDYE/wC0rhi1WG7hC3UFxCFAlMxqHhNSPttq46Sq +1S2kuqXAPAEyEjyHCg7Vd6T8omNOYLDf/WULOsDmzBMeyqzbYdidtdOi7SvQ2qSBpUJ+uufFLK3V +rQAlKjIA5DwpdqeZJoOu+pjtlh8Rtqd/WTWUr1N2+zyo54rLiuP9NI/ZWUDLrop1YAgaTu21B5fw +iv8AtXHrzMmNJ25iuzuuIytzLgUB3UtN/Hta5DW1J+jQRBZAMwTWBid441KlqTFZ2IkkpE0EZ7OJ +3FeKY7xhMVK9l4CvOy24cPKgiCx5D41qWoMQRUuWSREDjNeFkapIoEEi0Fro1Q4EJAIRzBnf69/S +lXrlpSrwtdnp1ksgtjgVSeXhtXpYT/NE16GeGw28qDxm4bLlmXez0hep8dn4K25eG1aBVsm00KIL +obUmQjiZOx90QeVO2WmFFKFMjUTx1QPftTj2FggkBgDURBe3+ygi7z2Zxp/sVJSlTiS03o3QnfaY +9PXjSeG+zNsuJfSky82RIJOkTq5c5G3OpP2JorCAlkd0GS5sd/t+6tnbW3aX2iW0KSFboS7Mj4UE +Y8m2DKezUndrTu3BCtczw8K2U1a+03L6XG9Dpc7NGg93+by293hSzrLanCW29KTwSTMV4liOVBHP +WiUL0ocS5wkpBiffSaWNztvUr2G86a9DMDYR40EX2BE6RNbJYPGNvGpMMA8q9LJ8B8KBghnfhSoa +QdwmDTzsiTukVuGjEQAaBu22EwIpw0gTsK3S0rbu07tmVTPCg606nrShk5bihA1OJHn3xWVKdUtC +R0bJUEgEPLB24nWr9kfCsoNOtVaOXeUEttNqWsplISkknStBP1Sa5NXhS9KdVncSAdXcO/hX0PxG +wssRt/Z7+0ZuWpnS6gKE+O9RRyblgmfka2HpI/bQcDDDGAynVZ3Xac1Rtz/7Ui9hqVbtMOpE7yJr +6ADKGWxwwlj4q++tGcmZYZWtbWEMoKzKgFqgnxiYoPn/APJrv/oufCvBhywf4Je3ka+hP4L4BEfJ +jUep++vBlfAAZGGNT6q++g+e3yavj2S/gaw4a4Y+Zc/RNfQo5XwAmThjU+p++vPwWy/M/JjXxV99 +B89Pkp4meycj+qa2GEvT/Aun8019ChlfAASRhrUnzV99bHLeBnjhrJ+P30Hz2ThL+/zDv6Ne/JFw +Zlh39E19Ck5dwVPDDmR8aw5dwX/49r6/voPnl8kvD+Idn+qa1+S3Z/gnI80mvoactYESScOaJPmf +vrwZZwEGRhjIPv8AvoPnmMLdH8W5+ia9+TF7y0ufSvoYct4EeOGMH1BNe/g7gn/xzP1/fQfPA4cs +H6Cx5RXhw9QH0VV9DXcsYAv6WGMn4/fSasqZdJk4Wz8VffQfPgWPPQfSsNlvOk719Al5Qy2RvhLP +xV99aLydlk8cIY+KvvoOChZ4fKtSbgbbcONYLO030h7yMV3j+BuWNR/ce34+f316jKGWwSBhLIB2 +O6vvoOE7e0swiXRcSBtoAiYP7Yq69FHRxdZ9xt2zs1qtLW3QHLi5WmQkEwEj+keXoTXXyMpZcSNK +cJYA4wCfvp5Z4DhNqoqtrQNEiDoWoftoGeRMrYdlDBk4PhfaezoMhThBUSSSSSAPGsqfSAlISOAr +KD//2Q== + + + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: image/jpeg +Content-Disposition: inline; filename="custer.jpg" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/4Q1kRXhpZgAASUkqAAgAAAAIABIBCQABAAAAAQAAABoBCQAB +AAAAyAAAABsBBQABAAAAbgAAACgBAwABAAAAAgAAADEBAgAOAAAAdgAAADIBAgAUAAAAhAAAABMC +CQABAAAAAQAAAGmHBAABAAAAmAAAAOYAAADIAAAAAQAAAGd0aHVtYiAyLjExLjMAMjAwNTowMTox +MCAwMDo1NzowMwAGAACQBwAEAAAAMDIyMQGRBwAEAAAAAQIDAACgBwAEAAAAMDEwMAGgAwABAAAA +//8AAAKgCQABAAAAyAAAAAOgCQABAAAA9gAAAAAAAAAGAAMBAwABAAAABgAAABoBCQABAAAASAAA +ABsBCQABAAAASAAAACgBCQABAAAAAgAAAAECBAABAAAANAEAAAICBAABAAAAJwwAAAAAAAD/2P/g +ABBKRklGAAEBAAABAAEAAP/bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAk +LicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIy +MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/AABEIAIAAaAMBIgACEQED +EQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0B +AgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpD +REVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmq +srO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEB +AQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFR +B2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVW +V1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrC +w8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEAAhEDEQA/ANNRzUiIfSlR +DmrUUfrXOajEiz2qZYPap0jqlrOr2+i2RkkZDMR+7jLYJPqfanYCd4OOlQGe2ibbLcQo/ozgGvPL +vxRqE6kSXsj+yfu1/JeT+dYsl55m7JUZ6kD+vWiwrnsAlgfG2eI/RxUhjrxlLhgMLKcdua0YPE2q +2mDHfSMB2c7h+tFgueqeVimmPiuO034hKWEepW4APHmw9vqK7G3ube9tlntZUliboymiwXECCgpU +oSnGPNIZEqgUVKIz3opAVUXDVajFV4/vVT8R6yNC0SW6THnsfLhB/vHv+AyaaApeKPF0WjK1nZ7Z +L0jk9RF9ff2rzC71Ge7naaeZpZWOSzHNVZriSaRndyzsdzMTkk1GFLHABP0qhEhnc5xWjpmg32q5 +8oY9M96hs7DzHG4MBjJJGK9j8HW1ta6ckqqHVunHQ0BY8wl8H6hDGWOcjttNYtxbzWxIcHivoXWN +X0nS7YzXh5I+WMDJNeY+KTb3umHUotOlhgkfYjkfqaLhY4ESZrb8Oa/caLqCMrFrZyBLH2I9fqKx +Y4mkcIilmJwABkk16R4V8BmDy9Q1dfnGGjtj292/woCx2qjIB9aftp+MUvFSMYFopxopDKCDDdK5 +L4i2V9dWdlJb28kkEJcylBnaTjBP68116ctWjanFUhHzpit7SLa0t1W7u5lCk4AxmvZbzwvoGoOZ +rnS7dpDyXUbCfrjFeZ3WkW0OuXEcMMbQK52RljtA9jTESWNvDqExntAJDu2YEf3R6/413Xhq7itb +IW88MgKMct2zmvN47h9C1WCS1uzGGbbIqnPynsexFelLMloUMjF4pVBEh75oAvX50q7+RIXknkwG +Pt6c9q0bu00e/wBEk0m/kht4pE2KrOAwPYjPcVmWlza2TSXTbWwMgep9K5zWdavNSmISzS4bOMjA +x6AUhm3pHhfRNFVZLG3EkmOLiQ72P0PQfhWnITUVjJNLYQPOmyUoN6+hxUjZzQAwr7UgWpKCMUgG +7RRRzmikxlGNavwDiqiDB6VegGQOKpCJygkhaM9GBBrzHxFp50+5la8tnO4DbIg6LnqP0r0S/v5L +K1MtvaS3bZKbYcHDDqD7+3WvONT1u+1W/jbUSVtY3yYEGAB396YjCtLQXt1mOBhAjZDPyT7V3GkX +bJbtp92pe1I+V8ZMf19qzta8T6dounx6dplhbySkbi/U4PIJP9K4y88Q6reYMk2yPOQiKFU/h3oA +73VLC6trFrmCVJ7dTn5X6fhWLo3ih31i2N3Bi1iYqWjHTPc+uKg0zxCb+2NrNaHj75hOFI9wKs6j +FF9hf7CotmjAZWTj659aAPUQVdA6EMpGQR0IpjCuJ8GeKVNtJYXshd4xuiKLyw7jH9K7SK4guGdI +nBePG9Dwy5GRkHkUDFA5HWnMBikIwaKkBo64FFAPNFIZVQVi+IvEk+mSJa2OVnxueQKDt9AM1uov +NedeIJkufENyySEAEIMHjgY/mKpCZaOri/hjg1G3jnhRtylB5LofVWXHP1zWpqF1pt1pg8+4jmlS +PbA6o32lj/02ydpGOMjr+lcsXliHOJF7jHNQSyAYmiPyA5I9DTEQnTVimMwywHOD6VoafJHp13Hc +pZW92gJIhnGUORiljcSJnqDUMTBQ0fdf5UAaFhANDj1STUbP7PJN+9EQHHlnJUKPTJNSW4G4jqCB +1qKKzu0EeoXV0k0N0CqiSXc4C5HI9OaSNsXCIvCg4oAzNbj1O01iDVXhS3km+dPKUKpHTOB61b0j +xNLYayl3JGBE6COWNOAQOhFMv7QGCa7N5E489kFuZCXTHOcdhWMGBJJ6CgD3BJEliSVTlXAZT7Gn +AA1xvgnXJrxX0+5fd5SAxEjnb0wf0rshjFSMTABopwxRSGVgDn615TNCTNMrctHIwP516xzuGK8z +1FBa+I7yJgNrynH48/1qkJleOZo0GMstMlDy5MKxsx6g8GpGUxSnb9084p5RZUyuFb06UxGfazPE +WicFWB4B9Kk3fv8Ad68VDOri4BaNt443D+tEchKsfegDQsVQy3DjJcR/d7dR+Va9lDGLR/NkLOzE +gAenTn6iudsJtmoMeoaMg+3IrbtpJVjXa+O/NAGK4Q3M5bG7ewz+NZqAZcdlY5q3bxXOrasba0QN +LNIxUFgo7nqa1PCPhttZ1a6S7Vha2rnz9p5Yj+EGgDLsZ7i0nS6t5WikU5BHp6V61o2pxatp8c6M +vmYxIgP3W9KI/Bukyu0stmgZkwkSsVWMepx1NcOss/h3Wp47K4WRUfa+4cSD6UmM9IGM0VQ0zVrf +VLbzImxIB88Z6r/9aipAn3ZNcB41tGj1gzKMGVFcH3HH9K7pDzXJeNCrTRuzHem1AueMNuOfzX9K +pAznLa7juUCSEJMvr3qZwydRx6isi4ty3zJw1TWksjrs3kOOoJpiL0twwXlCwHoKxri6WFj8pG/k +A1fe6vIODtKmu1+Gc9vcazNHeRxyZh+RXjVuc8nJ5oA4zwxp91rmpSW1nGGnZMZbgAZ5JP5V6Cvw +213Jb7TYgbcKPMbr/wB816pGsESARpGi+iqAKT7VCPlMi5+tAHisPwc8SLJk39hH/tpI+R/47XWe +FvBeqeGtHnhlENxO8hlYRSH5vQDIHP1xXoSzow4ZT9DSvIO1AHiepeMNUWaWCBPsgDFWDrl8jsc9 +PpXMajqdxe3BuLt90zAAsABkDp0rq/ixpTWeuwanbDC3kZDr/trwT+IIrziS6uUcJcqGWgDpPDOo +pb+ILctJxITGQffp+uKKyNFtjc67YxqdyNMrfgDk/wAqKljR64vB5rK8Q6DaapCtxI0qXC7I0Kth +eXAGR3+8fzrTDYNQavKRol465DpEZFI6gryP1FJMbPMEcqxRhytNkwrLInDDriqST/ay3z4lB3A+ +tSrIWQq3DjqDVEl9h9oiDIC2e1V4H1KxvEuIGe2KHPmZxtqkbuazYtC5FVJ767vG/eyEr6dKAPQr +P4k6tZRHcy3KKeGlXJI98VuQ+OdM1EB3nubN3IASWM4Y+gIzXksN4YcB49yjsank1L7Q4ZvlVfuj +3oC57To2vQX8jJBdJujO0hnCnP0ODXWxC6dBuLL6ZFeP2/izwp4hsIodejmtNREflvdRICjnszY5 +OepGOtcvc6vqeh3Ij0jXbnyifl8mZth+maLDPU/i9Cf+EOt53YB4rtcHvypBx+leILdgDbICwrqL +7TvHPiVo7fUTdTxp8y+dIAg9+uM/rXS6D8PbDTQs+pst5cjkJ/yzU/T+L8fyp3AyfAmiXHn/ANrz +I0duqkQhurk8Z+mM0V6BI+FCqAFHAA6CiobA/9kA/+EMRWh0dHA6Ly9ucy5hZG9iZS5jb20veGFw +LzEuMC8APD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQi +Pz4KPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUg +NC40LjAtRXhpdjIiPgogPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5 +LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgog +ICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICAgeG1sbnM6dGlm +Zj0iaHR0cDovL25zLmFkb2JlLmNvbS90aWZmLzEuMC8iCiAgICB4bWxuczpleGlmPSJodHRwOi8v +bnMuYWRvYmUuY29tL2V4aWYvMS4wLyIKICAgIHhtbG5zOnhtcE1NPSJodHRwOi8vbnMuYWRvYmUu +Y29tL3hhcC8xLjAvbW0vIgogICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50 +cy8xLjEvIgogICB4bXA6Q3JlYXRlRGF0ZT0iMjAwNS0wMS0xMFQwMDowNzoyNyswMTowMCIKICAg +eG1wOk1vZGlmeURhdGU9IjIwMDUtMDEtMTBUMDA6NTc6MDMrMDE6MDAiCiAgIHhtcDpNZXRhZGF0 +YURhdGU9IjIwMDUtMDEtMTBUMDA6NTc6MDMrMDE6MDAiCiAgIHhtcDpDcmVhdG9yVG9vbD0iQWRv +YmUgUGhvdG9zaG9wIENTIFdpbmRvd3MiCiAgIHRpZmY6T3JpZW50YXRpb249IjEiCiAgIHRpZmY6 +WFJlc29sdXRpb249IjIwMC8xIgogICB0aWZmOllSZXNvbHV0aW9uPSIyMDAvMSIKICAgdGlmZjpS +ZXNvbHV0aW9uVW5pdD0iMiIKICAgZXhpZjpDb2xvclNwYWNlPSI0Mjk0OTY3Mjk1IgogICBleGlm +OlBpeGVsWERpbWVuc2lvbj0iNzU1IgogICBleGlmOlBpeGVsWURpbWVuc2lvbj0iOTMwIgogICB4 +bXBNTTpEb2N1bWVudElEPSJhZG9iZTpkb2NpZDpwaG90b3Nob3A6Zjg2ZTcwZTQtNjI5OC0xMWQ5 +LTllM2YtZDQyZjM0NjM5ZGJiIgogICB4bXBNTTpJbnN0YW5jZUlEPSJ1dWlkOmY4NmU3MGU1LTYy +OTgtMTFkOS05ZTNmLWQ0MmYzNDYzOWRiYiIKICAgZGM6Zm9ybWF0PSJpbWFnZS9qcGVnIi8+CiA8 +L3JkZjpSREY+CjwveDp4bXBtZXRhPgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +IAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAog +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAg +ICAgICAgICAgICAgICAgICAgCjw/eHBhY2tldCBlbmQ9InciPz7/2wBDAAUDBAQEAwUEBAQFBQUG +BwwIBwcHBw8LCwkMEQ8SEhEPERETFhwXExQaFRERGCEYGh0dHx8fExciJCIeJBweHx7/2wBDAQUF +BQcGBw4ICA4eFBEUHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh7/wAARCAD2AMgDASIAAhEBAxEB/8QAHQAAAAcBAQEAAAAAAAAAAAAAAAIDBAUGBwEICf/E +AEIQAAIBAwIEBAQDBgUBCAMBAAECAwAEEQUhBhIxQQcTUWEicYGRFDKhI0JSscHRCBVicoLwFiQl +M5KissJDU9Lh/8QAGQEBAQEBAQEAAAAAAAAAAAAAAAECAwQF/8QAGhEBAQEBAQEBAAAAAAAAAAAA +AAERAjEhEv/aAAwDAQACEQMRAD8Am8Hl60dEGN+tAbnalEBIr5+PUIoGcmllXPSgkW9LxxntVBET +2pVY/Y0tFEe+KcpFRDYRDHSjrD/pp4sdHWIUwMHhwOlJcpXr0qVeEEd6aXCxxRtLK6xou7MxwB8z +TDSSpjelAtQFxxvwjaSmGXW7dnH/AOtWkH3UEUeLjjhGXATV0z7wyD/61cNTTDbG1AKcdKjBxXww +RznWbUA9Mkj+lSem3+n6lE0mnXtvdopwzQyBgp9DjpTDXGQ0mY6fFD6Vzy8DpTDTMR7UOT3p0U9q +KF3oaasm1F5PSnTr3pMAb1FJrGwGRRuX1o64xvRtqYG5X2oyp7UpgV1agCKAQSKFKZHLQozUEoNL +Rj3oBd+lLxJsKrTsaEmnccW+d65Ehz0pzGp9KuI5HHv0pdU7Yrsa+1LqmSBitSAsabdKrXiLxfb8 +IabG6xR3V/O4EVsXweXuxx2/maX4w410PhmJknlWe7A2gRhkf7j2/nWCaxxK9/qs+ohea6mbJuJd +2A7BR0UDoKuJaud54gcYX0Jk/wC46Lbno5T4sf8ALJ/Sqlq+sx3j/wDiOoahq7A5HmSFIwfYf2qA +uLxpn55ZGkf1Y5ptLPynAxVxEsdQXcwWNrCvuvM33phM8jS+Z5rZBz1pn+JxuWxRGvxzbDI9aYbE +rHfXgGFuX/T+1OrPXdZs3LWt/NCx6lTjP2xUGl3EO9HN3Gds/ah8XnTvEjiiywZLsXKj92UBs/ff +9at+heL9jNiPV7FoG7vCdvsf71izzAocGkWkyD1phuPVuh63o+uRc+mX8NwQMlAcOPmp3p8UA7V5 +Gs725sp0mtLiSGRDlWRiCD7EVq3Afi5KrR2HFA8yPoLxB8S/7x+8Pcb/ADqYNfkA3pLlpWKWC6t4 +7m1lSaGReZJEbKsD3BoKtYWEwvSjhQe1GA9aOoFA2K4JoAb9Kdcq5zQEagnYVKpALkHrQpyAB0Az +6UKioTlIpxCMikyMiloU2qwLRfMU6iGe4puin0p3Ah6kYHf2rUQsgVFLuyqoGSScACsj8TfFURPJ +pXDEo2+GW8Hf2T29/t61D+MHiM+pTy6BoMpFhGeWedT/AOeR2B/h/n8qyiSQjuc961ImndzdzTyN +NPK0sjHJLHNNZLk5601klY96SZqrOnRuipJApKWdn70iuTTqztWnkUAEg+gqp9JRrJIQEVmPsM0s +LG9dsC2lyenwmtO4D4PZ3jmKeYp64GQPnWoDw8WeBZIrZUbqMj+dTVx5l/yvUACfw0gx1yKQaOaM +kNGy/MV6UuuALhIyGRDj2qj8W8E3do3mCDIJ3wOgppjICzD2owmPepnWdKaAtlSCO2KgGBBwe1VM +OBJnYUA2BTdTR1zQaD4UceXHDd+thfyNJpE74dTv5JP76+3qO/zr0PHiQB42DKwyrA5BHrXjtc16 +l8Lo75OBNIF+GE34cYDdeTJ5P/bisdRqLGq980cLjpXQpB60dQc1lonymuBd8UsFocveoogXahSm +2+1CsqglI6E05twu1NU69KdW/XFaiHcYXOKznx44xOi6QvD2ny8t7fITOyneOE7Y+bbj5A+orRlK +Rq0kjBUQFmJ6ADqa8l8ba5LxDxTf6vITi4lJjH8KDZR9ABW+WUY03KMCkC5cnJorHJoA4rQ6elBI +2kOFGTjNGUcx32qZsdAubiESoC+eyjcUQwgspVTmZod+ilxk1a+EtC8/UIeZ+WOVSSmNw3/WKjG0 +60tpbeCZJTLIQVKHIO/Tb5Vb7Vbhbxr+1jES82Vjkckuo/d6bHHr7VBtXhrp8ttaxhreGKFfzcw/ +X1rTbdomQBQCpGxA61ROFW/EaBDcsQrcgyo3GcdKnLNTHmWV0VTjdTjb5f8A+UaieFjHK/mHBQZJ +J6Yqr8XXPD8UDie9tkC5BzKPsKrfGnFGu6jM2k8OlxynlaY/EgPYbd/sKyjTOFeK+JeJmtdUSRgk +gDM/wDrjI9fuaiadcY22k3EL3MbRgcpCuB136VjOoIou2ZNlJyK9NeKfhUR4fwS6aZDLYEtcIDu4 +I3b6YrzJdczTNzdtvtViEQoNKwxs7qiKWZjgADJJqS4b4e1fiC/Wy0ewmu5m68i/Co9WPQD3NejP +C7wtseEuTU9UaK+1gj4SBmO3/wBuerf6vtVXFO8K/CR08nXOKoimCHgsGG59DJ//AD9/Sti5eu1O +5zzHem/b3rPqyCgHNGUUXau5qK6a4a4DXcZqApOxwDQoxWhWK0gsdDinEGAelJMgxS0CgVqMmHHM +7wcDa3LHs4sJsEdvgIryU29ez0tobmNre4iSWGVSkiMMhlIwQazXjHwGtLsyXfCt8LWQ7i0uCSny +V+o+ufnXSMvPGN66BVl4m4H4p4clZdW0W7hQHaUJzxn5OuR+tV4Ieb8pG/pVE5w/odxcSiaWNhGm +CB6g9/lWn6XprWmnk5jifPQ9MHPX5AVQtK1trTTrlDkyOnlg+m2/9BTF77W7iN3Ms5UAFiW6elRF +6gsNQZ31aOS3gnjCrIzDnJU9cdh9KXKZto52hYWkcgAxj2OGx03qF4D1qOyv4oNSvVCznq+8cfpz +fP8AStj/AOxtla6VJDaXLXMDp580gwU5zvzAg7/agsXBN7pqWqWYYK4QOQTsexq4wjTrpArMvKw+ +1ZXw1oyxW8byOfPyc4bOKmJWvLRSPMbH5gcYxmoq5axacP2ulSAzx28e55om5Wyfl3qJ4Jt9Gtbt +ruCGeaRsBGlkJyM5O9VaOG8v7oJzFhnY9Rg1oGh6BIlp5WQVC9em+KKstmq3isvLEI5MgqNwfXtW +O6j4EcH2/Fd7qF811PBNKZorQNyRIDuVyNzv7jtU5xBxbxhoDfh7Hh6wEEPWV7gF3/2ov9a5oPE2 +rcUomoXUFvBaRgoAHLSM/cEY+ED077VUSem2NhpNmLLS7K3srdOkcKBR+nU+9CQk96OxBpN8YqVT +dzSecjpSjgE7UFUBTtvQJMoAz61xRntSpAoBRisqIF36Cjcu3SukUBt1oOcu9CjKAaFYq6r2AcZI +pxEvTem4SnMKYrUZPrUbipa2bG9RNqN6koelbglElBXBGR6Go/VLezNlPKmnW0kgQkc0S7n7UvET +iofjLVrjS9PDWyKZHzueigdT+o+9VHmziHSiuu3d6UjJmdZ1B/KBJvnHrnI9iDUqAlxZLplxp6lF +AdnXClhk7j164ovF/Pa6v5v4ZppJCWl5JCNic8uOg3Pak5+JntoRaS2DRyQrkCQbgEk9vcn70ZV/ +XtFktLCS4ihcRscDKbjpt/Or34IW8eraeLS4vrkJbyeYsAlbkLDcZXoaoGtahf3/ADT80nk8u4J+ +EbdKmPCrU7jh/VI71gTaSEeZj933+VFbglytnfGKdVUMxOMGpaSS3uYEBQZGQDmm2o2FvxBpy3tn +MAWXmSRNwfqKqY1HUdKumt7wFcbFlG30qYrQOHohBdBlwA24wOm+f+vrSfGfF8nOdN0+8MKQjMrq +uSx9B/aqmeJkS2llRsyKhKgnHNjNQcU97qzNGzRKZy3MScg/P5bUDbiHjp7GZFt9OlupWBBeQkkD +tnHqMnFO/Crij8fxBLbRwGOKaIyEZ3LZzzH7kU2nsI9Jljt5ktnuZRhZMZ39fUbZH2rQ+H+GtN0x +zeW9nDFdSKFkdR1wP0+lBNk5FEkyRSpGBik2HzqBHFGA964QAcZoZHTNWtDADHWuMpxkGuLgnFK4 +GOlZoR5cbmisTncUs4xtSRHM1AF+VCjqu2TQrNVAxg9KdQj1pJBvTmMHpWoyc2y5bYU+jUimtuu1 +PIx9a0HMewFR/E1g2oaXJChxJjCt6Zp+uwBpxFvjeqjzxfF9O1r8XeIwYebHKh/ccMcbf7Sp981A +Xer8P3Mt3eXrPJMIisUUSgZboC3etw8W4OFE0sXWs31tYXiDML8vNJJ/p5Ruw/lXn6/01dXuGvtO +tMO8uJQgPIpYZH3wftREUbt7pvwFkk0dtLgSK+Pi9/b9auOkaekdqEIxt27Va+A/CfWbuwTU4tPE +kL5BmdwoGOu27Y+QNPf8hcSyxxhAiNy87HkUn5tj+9Ax4M1274duxDcRmfT2OSq9Y/cf2q7cQaVb +a1apcWEsTLIAUYbA59ao2qXXDehoX1PU43cDaKE4P1JGfsPrTXhjiXUtUMknDn+W21ssn/kTO3O5 +9cbkfeinXEXDuraYhafSJJ0XOJIGz+lUK/4ku9PZ1tw6GPJIdcEe1bXJxWLfTHg1WExl15SVBIBP +p9axbibSdT4g1aX/ACyyldJGCmRl5VC+pJoENB41upOIrK/1W1W8tYHBeHJGR6/Mdd69P6Ve2Wp6 +bBfafKJLaZOZGH9fevMs3DtnoFsJNTW6ZCcGeJQVQ+46/fFX3wr4ls9Im/Cxail1pc7ZYdHgb+Ir +6euD70I2Vl+HNJkUvsyBlIKkZBBpJhRSLYwaKuD1FKkY61zAA6Vm0EQKW2pZeXuKSU4pQnbaoorh +TSZ5RR96IwqDqnOcChRosAdaFZqocYB6UrG2/Sk1BzS8Y3rcQ7tj7U9i3ppboaF1r3D+l+bFf6xY +296q/s4ZSTgnozKu+PatodalfWmm2Zu76dIYV/ebv7AdSfYVmPFnirO4ktOHoDD+7+JlALf8V7fX +7VfOEtW1K0sNSng4j0Xi7VrlwbeGS9NmkSY/IkfKwXt8+5rJ9S401nT9TOleLPAyzo7HlvIIBDOo +/iR1+GQD2OKuMqNftPqd1JPqM81xNJ+aSVizH61evCS+s40l0q5jhWYDy/2oGCMgxsc9gwAPtUzx +F4ZTDQouJeGmm1TRp4xMvNHyzRoRnJHRh7j7Vmmr+ZYTW+q2p/aQNhwD+de6mojWPETxMvdE0KG1 +1LmFxFzJDaKRGWYE/E6rsFHQD296wHiDjbiTXJna71KZUb/8UR5Fx6bdfrTXiC9u9b1iW/uA3NId +lySFHTApyugO9mLiHJI3xVVXzzN8TMSfc0406/u9OnE9pO8TjuO9WCLR47q3DACGdR3GVb2P96tX +iN4K8Q8G8LR8QXV5YzxYX8RBEx5oS3pnZgDtQJaD4j3z2fk6laC5jGzSR/mX3q26JxClzah47kTI +35SVwe+29UfwY0NNQur2/uA4ihVY0IPVzv8AI7Dv61aNStX03U5YT5fKeZ4+VcdOQ7j1xn7VApqL +/wCZWNzbSr+zmjZcHsazC202WwnVortlukJ/J0BHbPetK06QvEzEHIkI+hFI3fh1+O4e1Pia31uO +G5t5ARZsm7DAyQc5/Sg0Dwd4ztdU4Y/C6jMyX1rJycnLn4NsfYnGPf2q6aVq2m6tC02n3ccyI5jf +BwUYdQwO4Psa8kcLa3LpGssWeTyXfEgU4Jwa2vWLxdQ4e1oWGnabc6Zcwfio721YJcQuF5v2i7MQ +G2BGwzvRZWsOuKTIIqgeC/FjaxpB0q+maS+tF+FnOWkj9fmOn2rQGO9ZsUl0augkVwjfOKAB9qiu +52pNmzttRiDRGU5xQGQ9xQrqKcbihWaIxF70snWklB74paJcmtQRXHHEq8NaA90ih7uU8lup9cbs +fYf2rDLvWJ9QvGur6WWWd2y7Mc5q9eOV60Op6XbqhYLA8jHtu2P6VQYruAkeZFsepxW2KfQTI35G +Gau/DvG99aWo0zW401vR2xz211livujHdSO3p7VSLdrGX9/lPTI9KXaCXk/ZOjemaDW9M02NpbTi +LT/FjVrPhaxbzf8AL7mTnmgYDaH4iQVxtuDt69aoviLrXD+u67LcaLpE1pBJnzWcgLMf4gmAVPr2 +/nVMnlBcRXCBJ13QkbA06065FxCWcYdG5WT0IoGI0eISMgUe2RR9OJt5Ws3xjtt1qUB518zG6HtU +frScskVygIwd8UHZ7copUCjaxe8bcR6Yuj3V7NeW5OUiCc0jY3+ZG2aXibmUSb4YdRU3wbxqvA2q +zak1i9z5kBjDkcxjOf5f2FTQXwqnWz4eOmvGY5orhxICuG5tuvv2ol9rcOs69d+UcpZOsRYfvFld +f54qNtOJ5+Jtf1XVpMwySzI/L3/Ly/8A1FHg0aLS9aklslKw3flM8Z3AbzR09tzVEjYcoeULsGAc +CmvEOnarqt1aWelreSySK3PBbgnnUYO4HYUdJkSWAKd+XlOPepJ+J9S4VhttZ01Od1k5JEJ/OpBy +PuB9hQZLxHpk+n6sySQvE2fiR1wVYdQQad2F5eWkb/h5XiJjaNgDsVbZhj3qa4o1W64nu5tVvIDF +M7c2CcmohlBJ2xlaB9wzrl1w/r8Go2ZUyxDHK35WB6g16R4T1634i0SHUYF8st8MkZOeRh1FeW/J +WRw2enU1qvgnxTp+nR3Gj6lMtuZ5Q8EjbKSRjlJ7dBUpK2QkD0rg3zRGIJ2O1GSstlFwAc0VsA7V +35UXBzUo6p2oV0KaFZVGKm9LxKM9M0T0NdklENvLLjPIhYD5CukRjfi1qCXHF88eQY7ZFgH2yf1J ++1VyGOORMYGKbNK97JPLcMWkkcyOT3J60haieGQqmeXO2a0wkJ7FGHwkKexBpBLqeycJOTy9m7U/ +tJo5sebGMinM8VpLGedMqN996COvF/Hw+ZFyyOo6A9fl71H2NwEvg2cGQcrjpuOmf+u1PxZRK7SW +NyFx+5UfqyK3/eEBW4jPM4/jH96Im7d/jOGG/auXo57RlO/ptTHS7xJ4lIOT0p60mQ4GxopppdwG +QwOfy7UtIshlFuInmD4AULnOegqOGI7wMo2JwfSn8mXQgNysR8LZ70Fp4h8ML/hWK01C51Czb8eq +q0ETEtGx3HsR7im96wjWJFlVmjKq3fmxlz/8ahtK1TXdc4jjt9cvGljit2EQXbmIAAz9KXnYWVq8 +apjCSSDfJGwTr/yNAiMlFy/7oORvjepDihUk0CGFZFjDToDI/wCUZOMn7060vT7OXQVvpLadCwZY +wzBEkPL8JJPQBs5OcHp61E8Xs3+TwW867mQcy47gGgl+P+CtL4d0qyv9L1w6glwgLq6gEZHUY/lv +Wb3MmG5c4PKR9akYbaRinPczyRpuiO2QufTNRuqxNHdKvYsD/wBfagPFtEAegFFkbzcKq7A9aDDK +igx8qHb8zHCiiPQfhJrX+bcJQiWQyT2rGFyeu3Q/aropX0rzp4e8YS8JXqxPCJ7O5I89R+ZcfvCv +QOnXttqFlFeWkgkglUMjDuKzY1DvPegGoLRWqNFFbI7UK4g2O1CshgAcelGRFkUxt0YEH610jAo0 +LKD71qDzaI1g1Ke1JB5XZM/I4o1xGY2Db4FLcWRfgOML9OgS8kH0LH+4p75AmgDcwwa2wYYLplNi +BtiixzzRH4gWA7EVyM/h5miz8NPIRHMOUAADvnFENpYrO7X87wSnoU/tTC80bUoR5kEgvUG+xww+ +h/pUxJawr8cec02FxJE27HlHrQQOkTG2lliZCuGyA2xHtUxFKzfER9qYaxBLPci9tD5g5cSRj8w9 +x6ijW8+YsZHTegPLvuMDBp2JAIlbbcY6U2tpPMV8/LOKAceXjB2oJ3hCeN9Z3UMywvjbp0rt9bmS +OeeQZeQQrGN8AkFjt26ioLhed4uKIU5iOdXBH/GrLqDfEkaDc3BJ/wCKqv8ASirgbmxtLENZ6W7x +ae37cMQFYc+Qq8wOSAASfXvvVP8AEdndhLJzeZ+LfmBbJyc7VJPdW0ziW6hilkAzzFFBHyqJ8RLt +brTLec8gkacFiFwT8JoIK2lXlBYbVE8RMFaKT19/ejG5wAq0x1uVnhiB3IbagdwqHVc9MUSP9vdF +v3E+FaPptvdX7Q2NhDJPdzEJHGgySTWq2/hOukcMvd6teSC/EZZUhwY1OM4JxknttjrQZqYlDGRx +sOgNaX4M8Sw2Usmi6hOscUp5rcucAN3X60Xh3wf4k1e1ju7ua105GHMsUuWk/wCQH5flnNUnUrFo +NVms5MB7VzGeX+JTg4+1QemVwRkYxRGJzWd+H3HtrJDBo+sytDdKOSOdz8MnoCexrRRg4IIIPTFZ +aHRtutCiEkChWappJkLjJrkaEnJo8rCgjYxvW4MN8XrNYeNr3GR5ipJ09VFRWg3XnQlMnmTYg1dP +G+z/APGbO8HSa25D81J/oRWXQztYXvm78j7PVYqw6hFGSJOXp1OKbpzKOZExipGIxXduGT4sjam3 +lmJzG+du4NULW84YBZCPfaiXtosykxt13xXFCEEDOR39aNDKUk/KceuelEVq/gu4JcRrICD13okr +XBhE8x+IHDEdSOxNW55oHT4lXPqKjr+2je0kjHLlxjtRURZNiPYjfvXWbAIYCkbZuVeQnp1HvSU8 +yDIBG+2KIW055DxDp7W7AP5uCcA/Dj4v0zVnhdpZIWOxMZkP/Ni230IqrcMRo2tyTqCTDbyMBnbJ +HKP/AJVaLGRBcXDkhkiYRqADsFGB/KosSyxQOg50JPuKqvH7CJrG2RsjDyN+gFS9veTT3W4+HPrV +X48uObWljBwI4gMDsTRTbhvTbjXtfstHtZooprqURh5Gwq+5qa8ZeCLjgi6s7d9VttRSUkiSIcpV +h1BXP61VrW4ltikttIYZo2DI69Qw70e9utS1/VrSPULlp5ZZVXJwOpAJrSN7/wAO3B6Wul/9qL5R ++JuU5bcMu6Reo92/lWo2GmS6hqBv78AJGSLe3JyFH8Te/f2+dNOHEkt+FtOtRcRmNY1AePBGw2pT +U9RFoHSW9htI+txcu3KIl74z1JOwHr8qjUTkY/Fo0Vs7RW6bM46k98e9Y54+No6alYWtjbRw3MMZ +82RVwFQ9Ax9Sd9/ep3WvFzQ7G3FhollcXap8IkJ8tSPXfcn3xVP408QdP4k4Yk0M6GLbmkEvmGfn +JcHOTsPehqhpb2kjiTl53H72dqv/AADxs9hyaZquTaDaObmyY/Y+1Z000iAsRGF6DkJOPnSE1xME +ODzKR2rNiPTqyxTQrLDIskbDKspyCKFZ14IX7XGgXFm0vP5MpIBOcA9PpQrNaXlmwBtXRkkGkQwz +SqZyKSireMVh+I4TivAPitJ1JP8ApbY/ry1h+pW3OTgnHbevQPHsN3qOjwaDYcguNTnEILdAoBcn +/wBtYMRzKFIww2atxmmGl6nPps3lOeaPO4qz2t5aXyghwD71WLy18zJC71FwzSW8uCSKrK+XFpyn +nifJPXemp8xdgjZ+RqOtLucqG5+YEbUs2pyoCWJoHEqyn8sbkkdlNRV4uoKwPkuvtykVIwcQRK2G +yKPNrEEzpySFGXpk/Cw9DRVT1qeW1lWQoyCUZx03HWo1LtpG3JrdvBySz1fxAtbDUeG7PUomVjI8 +8SukAxs45ts5GPXrXp+z4b4chAaHQ9MjI6FbVB/SkMeG/CzTdS1jiRNOtdPu5TduiGZISyRKrcxZ +j6bV6l4a8FuDLSxWO7hvb+U7vJNM0ZLHr8KYx+taqkcUIVI41VewUYArsigSZGKuKoKeEHAMJDpo +rhvX8VKf5tTW78DfDW+na5udDleV8Zb8ZMOnyatLyCu/pQQgDqKDNbPwJ8M7W6juU4e8xozkLLcy +un1Utg/WlX8EfDsasurW+hm3u0bmUxXEiqp9lzj9K0gkAda7nK7VRVZeDdIeJwqzwzH8s6vzPGfU +Bsr+lecPGPh/ifh/iBbLW9Sl1GxkzJY3LfCGGdwR0DDv8/evWb7HOdqpvjLwsvGHh9qGnxIDf26G +4sX7iVRnl/5br9amFeQiOQ4yCPbrTeSLLl4zv1371DQX8yA+YCGHX2NBuJCshR0UgbdKiHdxd8jl +dwfSm348xnm5Oai/ibe/bflRj0P96ZXHNBLyMpA96GtG8G9cS34rWDIWO9Qxuv8AqG6n+n1oVV/D +S3a7490qNCwVZfNbHooJ/pQrnZjUejBjPSl12IG1Nhkt0pxGD1NSVcQ3F17HpeqcOalNJyRRaj5b +tnAAdGWsIlYJql7CSMpcOvX0Y1vvG3DZ4q4Yn0qO4W3nLLJDKwyEZTnt7ZFYxxfwJq3DV9LNNfR6 +j+xWeeRFK8nM3IOvXfG9bjNRbJzHAxnGMUwurJJVbb608t5gFywJJ9KcZXBBGRVREabM1rP5E26n +cGp0Q288PMNjTC/tkki5kwCB2613SblsCF2APT50AvNMj5TIuB3z3qNSMBvi7Hap65TbP9dqirhV +QFsYOelNMW7wn1ltC4uspjPy28kgSTmOwB6H23r2Fpl+rwhieY8ucj+9fP1794mJBAxWhcEeJHF1 +vYCCDU3Fra/xgNk9lyRnHtRXtA3S8iuCppvd38ceDg7HtWD6B40W7Wwe9tQZlHxRBsBv9p7VcND4 +80ni+ynbTGkt7222ntJtnXPQjsw9xV1V8XX7U8o5XO5H8v70dNcsOflZ+VvQmszvG1Yc7RJzxk5y +N+tIQX8sbjmkhBO551II+9T9GNfTUbaTHJMjfI04S4UjGay+zvp5CpJBP+kbCp2y1GRByFmPfc9K +foxb2lDE7/ShDIomUZ2OxHzquG8nZQyinNpdyNgEYYGmmPH3jLoFpo3iVrulEGBBcmaEqNuST4wP +pzY+lUu60i2EBImVpPbvWwf4z7Q23iBpeoRLgXenAE+pR2/owrCo7iUOAT9KrIvJJbyfDkYNSkFx +Hd24jlxzr+U0QeVdDHMFIG5NMJk8p/gl+1X0aj4Eaf5vFNzeFdrW2IB92IA/QGhVn8BdNntOF59U +uYyn46UeVnqyLtn5Ek/ahXLq/Wo0LlAbY0dAc9TSJOD70ojjY1iNU7iyM4yKqfF8IvNa1KyfcT8N +3DKD3aN1dftirVDKp9Kz/wAbLq60pdO1ywmWN+SazfftIvp8s1uJWNluQq/NsRnrTtJkbBUjGKj7 +OSOeE2rN8QGVNNj51jNljzRnvW2U7JKrKQoyR1phM/lSK64z1NKQOsq8/MObrmkJjzty8h9M1BOW +E63UIQMOalZdBkul+J8DtvVTE01pLzwyMuPQU4bi3UUQJkEjbJFMNSl1wzbwjnurxQvcLTC61OGO +FdO08IkS+nf5moDUtbvr5irynHtSNoxiIbO9WRFnVWEBfm3A9abadJxE16l3pN7cWk8eyyxyFDj0 +yO3tSNpqSD4ZcYPWp231e0t7blgYA43FBYLbxG474Zt+eW/ttULYDrOhJP1BGftVguOK+MNV4cte +I7LheG4sp8q01rO+Y5FPxKy9j36dMVlVzfSX1zsMhdzWm/4fePLPhniSTh7XJUj0bWCFLufggnH5 +XPoD+Un5HtRZVq4J8Q+GDpbza/FrMFzAc3Kxwq/kL/ER1K57gbbVpPDHFPAuuMo0jjbT5XbcQzv5 +T/8Apf8AtTTirwj0niC9N7a6oulXirmKSABiT/qGd1x2715q8SOAZ9D4gm0y4jWw1INlMHFtdL2a +Mn8pP8JqZFe1raxkKjlaGZT3RhinSWLKdosZr532mu8UcPXLQ2uralp8sZwViuHT+Rqaj8XPEeKE +xJxfqnKRjeXJ+53rWJrYv8cd9YC94ask5Wv44pXch90jJUAEe5B+1eaDKSMgkGlNV1HUNVvZL3Ur +ye7uZDl5ZnLMfqaluDODeIeLLvydHsHkjU4kuH+GKP5sf5DJ9qvgghI4YkM2/vWreEXhhfcQSxaz +r8Ulto6kMkbfC917DuF9+/b1GkcB+EPDvDix3mphdX1Jfi5pF/Yxn/Snf5tn5Cr9PP22A6Vi9Lhp +NHFEkcEEaxRRqFRFGAoHQAUKTnkHPmhXOqTcnNBGNChWI0VjbvgVnX+IiMtwbZzKQDHeA/dWoUK6 +c+s1g0N2/mrMuzKas0YS/tQ7LguvN8qFCt1kwV2tJ/KU5XNO2cqSQBkj0oUKBheFmzvudyah51Oe +tChViVyOFQfenCxKRvQoVUJSRgHrSTMQ2ATQoVYsOIZ3gBAJpve3Ek5BJ70KFIJXh/i/ibQZEl0r +Wbu2KkEKJCV/9J2q73Pi3xPxRZ/5JqENhNPe8tsLqSLLICcbD60KFLCM+4gDw63cWUkjTpaSNApb +qQpIrWPD/wAMeG+IuDLW/vWvYbqUNl4ZRjrtsQaFCs9XIs9XHhvwa4M04CS9hudVl5sg3EnKg/4r +jP1zWhW8dvY2kdpZW8NtbxjCRRIFVR7AUKFc9tawSSViDTSRsnehQqKbyjLbUKFCs0j/2Q== + +Cheerio! + +--Multipart_Sun_Oct_17_10:37:40_2010-1-- diff --git a/testdata/testdir4/multimime!2,FS b/testdata/testdir4/multimime!2,FS new file mode 100644 index 0000000..84f85aa --- /dev/null +++ b/testdata/testdir4/multimime!2,FS @@ -0,0 +1,27 @@ +Return-path: <> +Envelope-to: djcb@localhost +Delivery-date: Sun, 20 May 2012 09:59:51 +0300 +From: Steve Jobs +To: Bill Gates +Subject: multimime +User-agent: mu4e 0.9.8.4; emacs 23.3.1 +Date: Sat, 19 May 2012 20:57:56 +0100 +Message-ID: +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain + +abc +--=-=-= +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="test1.C" +Content-Transfer-Encoding: base64 + +aGVyZSBpcyBhIHNpbXBsZSB0ZXN0IGZpbGUuCg== +--=-=-= +Content-Type: text/plain + +def +--=-=-=-- diff --git a/testdata/testdir4/signed!2,S b/testdata/testdir4/signed!2,S new file mode 100644 index 0000000..7e1319a --- /dev/null +++ b/testdata/testdir4/signed!2,S @@ -0,0 +1,36 @@ +User-agent: mu4e 1.1.0; emacs 27.0.50 +From: Skipio +To: Hannibal +Subject: test 123 +Date: Sun, 24 Mar 2019 11:50:42 +0200 +Message-ID: <87zhpky51p.fsf@djcbsoftware.nl> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; + micalg=pgp-sha256; protocol="application/pgp-signature" + +--=-=-= +Content-Type: text/plain + + +I am signed! + +--=-=-= +Content-Type: application/pgp-signature; name="signature.asc" + +-----BEGIN PGP SIGNATURE----- + +iQIzBAEBCAAdFiEEaYec7RdFk3UPFNqYEd3+qdzEoDYFAlyXUwAACgkQEd3+qdzE +oDbjdw//dAosaEyqSfyUMXjS++iJEeDIwKwO6AjEI0xCbJjHmxq93PA61ApE/BS3 +d/sKa1dsfN+plRS+Fh3NNGSA7evar9dXtMBUr6hwL0VTmm5NDwedaPeuW6mgyVcB +VNUn5x1e/QdnSClapnGd156sryfcM1pg/667fTHT6WC01Xe0sezpkV9l0j4pslYt +y6ud/Hejszax+NcwQY7vkCcVWfB9K4zbiapdoCjHi78S4YAcsbd//KmePOqn04Sa +Tg1XsmMzIh7L/3njkJdIOd9XctTwYEcN5geY1QKrHQ/3+gBeaEYvwsvrnqnVKqMY +WCg/aYibuXl+xNkPMcKHIj1dXA3m5MkL77RrxODiAYz0YkiQx1/DLZs8PV3IVoB4 +f0GGDqyiOwSmSDa4iuCottwO4yG1WM1i7r6pir22qAekIt43wSdwakOrT1IkS8q2 +o0VGiQtEPy27D+ufiw06t02Ryf20Q7i2YcueZxYeRBq41m11M41DJ4wH7LQcJsww +qG5iBOdwQFCTWpi1UrbbFjlxXXWvKMuIU+4k7nsamrEL4SDXmq1v13vtlcgJ6vnn +v7c9+MF7laqdfI+BYnlD1v/9LosPbFTm0hPdvK4yIOORp8Iwj/1PGzTOz6SCUxzA +kDu+Y+NN9/SM1ppStg1OikYPcfEXF8igWhuORwqcmpgHxVkIQ9I= +=wnkU +-----END PGP SIGNATURE----- +--=-=-=-- diff --git a/testdata/testdir4/signed-bad!2,S b/testdata/testdir4/signed-bad!2,S new file mode 100644 index 0000000..7a37ba9 --- /dev/null +++ b/testdata/testdir4/signed-bad!2,S @@ -0,0 +1,35 @@ +Return-path: <> +Envelope-to: skipio@localhost +Delivery-date: Fri, 11 May 2012 16:21:57 +0300 +Received: from localhost.roma.net([127.0.0.1] helo=borealis) + by borealis with esmtp (Exim 4.77) + id 1SSpnB-00038a-55 + for djcb@localhost; Fri, 11 May 2012 16:21:57 +0300 +From: Skipio +To: Hannibal +Subject: signed +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:20:45 +0300 +Message-ID: <878vgy97ma.fsf@roma.net> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; micalg=pgp-sha1; + protocol="application/pgp-signature" + +--=-=-= +Content-Type: text/plain + + +I am signed! But it's not good because I added this later + +--=-=-= +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +iEYEARECAAYFAk+tEi0ACgkQ6WrHoQF92jxTzACeKd/XxY+P7bpymWL3JBRHaW9p +DpwAoKw7PDW4z/lNTkWjndVTjoO9jGhs +=blXz +-----END PGP SIGNATURE----- +--=-=-=-- + diff --git a/testdata/testdir4/signed-encrypted!2,S b/testdata/testdir4/signed-encrypted!2,S new file mode 100644 index 0000000..a3910e6 --- /dev/null +++ b/testdata/testdir4/signed-encrypted!2,S @@ -0,0 +1,54 @@ +Return-path: <> +Envelope-to: karjala@localhost +Delivery-date: Fri, 11 May 2012 16:37:57 +0300 +From: karjala@example.com +To: lapinkulta@example.com +Subject: signed + encrypted +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:36:08 +0300 +Message-ID: <874nrm96wn.fsf@example.com> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +hQQOA1T38TPQrHD6EA/+K4kSpMa7zk+qihUkQnHSq28xYxisNQx6X5DVNjA/Qx16 +uZj/40ae+PoSMTVfklP+B2S/IomuTW6dwVqS7aQ3u4MTzi+YOi11k1lEMD7hR0Wb +L0i48o3/iCPuCTpnOsaLZvRL06g+oTi0BF2pgz/YdsgsBTGrTb3pkDGSlLIhvh/J +P8eE3OuzkXS6d8ymJKx2S2wQJrc1AFf1BgJfgc5T0iAvcV+zIMG+PIYcVd04zVpj +cORFEfvGgfxWkeX+Ks3tu/l5PA1EesnoqFdNFZm+RKBg3RFsOm8tBlJ46xJjfeHg +zLgifeSLy3tOX7CvWYs9torrx7s7UOI2gV8kzBqz+a7diyCMezceeQ9l0nIRybwW +C9Egp8Bpfb02iXTOGdE/vRiNItQH14GKmXf4nCSwdtQUm3yzaqY9yL3xBxAlW53e +YOFfPMESt+E7IlPn0c7llWGrcdrhJbUEoGOIPezES7kdeNPzi8G1lLtvT04/SSZJ +QxPH5FNzSFaYFAQSdI7TR69P7L7vtLL8ndkjY49HfLFXochQQzsqrzVxzRCruHxA +zbZSRptNf9SuXEaX9buO1vlFHheGvrCKzEWa6O7JD/DiyrE/zqy4jdlh9abMCouQ +GWGSbn8jk6SMTQQ2Yv/VOyFqifHZp0UJD59tyIdenpxoYu5M0lwHLNVDlRjLEwUQ +AIDz1tbLoM7lxs2FOKGr8QqbKIeMfL+NUmbvVIDc4mJrOlRnHh+cZYm4Z49iTl1v +bYNMYgR5nY7W6rqh0ae7ZOW0h2NzpkAwTzuf1YrSjNavd9KBwOCFtAoZhRwfwFVx +ju+ByHFNnf7g/R6DekHS0pSiatM0cPDJT05atEZb+13CRHHznonmLHi+VahXjrpg +cIUA8Lhjdfm6Fsabo7gNZnTTRxNBqUXKK2vJF/XLbNrH5K2BH2dCCmUNtm3yFWiM +DOzaw3665Y3S6MvZdyKpatbNrVoJdBpRgPxJ1YCSEituFUqHJBStay+aRb5fVkQR +w3+9hWw+Ob0+2EumKbgfQ7iMwTZBCZP4VOxkoqdHvs9aWm4N7wHtXsyCew3icbJx +lyUWsDx/FI+HlQRfOqeAMxmp8kKybmHNw8oGiw+uPPUHSD1NFYVm2DtwhYll3Fvs +YY7r5s3yP1ZnwxMqWI3OsExVUXs8MS4UTAgO+cggO7YidPcANbBDihBFP8mTXtni +Oo5n5v+/eRoLfHmnsGcaK8EkKsfFHpbqn4gxXGcBuHaTTJ/ZhbW6bi1WWZA9ExaJ +IeTDtp5Bks1pJvTjCDacvgwl3rEBM6yaeIvB7575Y/GPMTOZhawhfOxV1smMmTKI +JOWYb3+PuN2cvWetkjFgH8re4sRXq22DKBZHJEWYU8sH0sACAePnIr+pkrOtGeJB +t1zBqZUnrupH6ptk9n/AjbQ+XSMTEKu55gSjYLAYx1EHApx52QLkdh+ej5xCIVeY +6wS1Iipkoc6/r6F7CKctupXurNY2AlD4uQIOfD6kQgkqK4PY3hsRHQA+Zqj6oRfr +kxysFJZvhgt26IeBVapFs10WuYt9iHfpbPUBQUIZCLyPAh08UdVW64Uc2DvUPy+I +C+3RrmTHQPP/YNKgDQaZ3ySVEDkqjaDPmXr5K0Ibaib2dtPCLcA= +=pv03 +-----END PGP MESSAGE----- +--=-=-=-- + diff --git a/testdata/testdir4/special!2,Sabc b/testdata/testdir4/special!2,Sabc new file mode 100644 index 0000000..7f1de8e --- /dev/null +++ b/testdata/testdir4/special!2,Sabc @@ -0,0 +1,10 @@ +Date: Thu, 1 Jun 2012 14:57:25 -0200 +From: "Rocky Balboa" +To: "Ivan Drago" +Subject: currying and tail optimization +Message-id: <3BE9E653ef345@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT + +Test 123. I'm a special message with special flags. diff --git a/thirdparty/CLI11.hpp b/thirdparty/CLI11.hpp new file mode 100644 index 0000000..41027f0 --- /dev/null +++ b/thirdparty/CLI11.hpp @@ -0,0 +1,10966 @@ +// CLI11: Version 2.4.1 +// Originally designed by Henry Schreiner +// https://github.com/CLIUtils/CLI11 +// +// This is a standalone header file generated by MakeSingleHeader.py in CLI11/scripts +// from: v2.4.1 +// +// CLI11 2.4.1 Copyright (c) 2017-2024 University of Cincinnati, developed by Henry +// Schreiner under NSF AWARD 1414736. All rights reserved. +// +// Redistribution and use in source and binary forms of CLI11, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +// Standard combined includes: +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define CLI11_VERSION_MAJOR 2 +#define CLI11_VERSION_MINOR 4 +#define CLI11_VERSION_PATCH 1 +#define CLI11_VERSION "2.4.1" + + + + +// The following version macro is very similar to the one in pybind11 +#if !(defined(_MSC_VER) && __cplusplus == 199711L) && !defined(__INTEL_COMPILER) +#if __cplusplus >= 201402L +#define CLI11_CPP14 +#if __cplusplus >= 201703L +#define CLI11_CPP17 +#if __cplusplus > 201703L +#define CLI11_CPP20 +#endif +#endif +#endif +#elif defined(_MSC_VER) && __cplusplus == 199711L +// MSVC sets _MSVC_LANG rather than __cplusplus (supposedly until the standard is fully implemented) +// Unless you use the /Zc:__cplusplus flag on Visual Studio 2017 15.7 Preview 3 or newer +#if _MSVC_LANG >= 201402L +#define CLI11_CPP14 +#if _MSVC_LANG > 201402L && _MSC_VER >= 1910 +#define CLI11_CPP17 +#if _MSVC_LANG > 201703L && _MSC_VER >= 1910 +#define CLI11_CPP20 +#endif +#endif +#endif +#endif + +#if defined(CLI11_CPP14) +#define CLI11_DEPRECATED(reason) [[deprecated(reason)]] +#elif defined(_MSC_VER) +#define CLI11_DEPRECATED(reason) __declspec(deprecated(reason)) +#else +#define CLI11_DEPRECATED(reason) __attribute__((deprecated(reason))) +#endif + +// GCC < 10 doesn't ignore this in unevaluated contexts +#if !defined(CLI11_CPP17) || \ + (defined(__GNUC__) && !defined(__llvm__) && !defined(__INTEL_COMPILER) && __GNUC__ < 10 && __GNUC__ > 4) +#define CLI11_NODISCARD +#else +#define CLI11_NODISCARD [[nodiscard]] +#endif + +/** detection of rtti */ +#ifndef CLI11_USE_STATIC_RTTI +#if(defined(_HAS_STATIC_RTTI) && _HAS_STATIC_RTTI) +#define CLI11_USE_STATIC_RTTI 1 +#elif defined(__cpp_rtti) +#if(defined(_CPPRTTI) && _CPPRTTI == 0) +#define CLI11_USE_STATIC_RTTI 1 +#else +#define CLI11_USE_STATIC_RTTI 0 +#endif +#elif(defined(__GCC_RTTI) && __GXX_RTTI) +#define CLI11_USE_STATIC_RTTI 0 +#else +#define CLI11_USE_STATIC_RTTI 1 +#endif +#endif + +/** availability */ +#if defined CLI11_CPP17 && defined __has_include && !defined CLI11_HAS_FILESYSTEM +#if __has_include() +// Filesystem cannot be used if targeting macOS < 10.15 +#if defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 +#define CLI11_HAS_FILESYSTEM 0 +#elif defined(__wasi__) +// As of wasi-sdk-14, filesystem is not implemented +#define CLI11_HAS_FILESYSTEM 0 +#else +#include +#if defined __cpp_lib_filesystem && __cpp_lib_filesystem >= 201703 +#if defined _GLIBCXX_RELEASE && _GLIBCXX_RELEASE >= 9 +#define CLI11_HAS_FILESYSTEM 1 +#elif defined(__GLIBCXX__) +// if we are using gcc and Version <9 default to no filesystem +#define CLI11_HAS_FILESYSTEM 0 +#else +#define CLI11_HAS_FILESYSTEM 1 +#endif +#else +#define CLI11_HAS_FILESYSTEM 0 +#endif +#endif +#endif +#endif + +/** availability */ +#if defined(__GNUC__) && !defined(__llvm__) && !defined(__INTEL_COMPILER) && __GNUC__ < 5 +#define CLI11_HAS_CODECVT 0 +#else +#define CLI11_HAS_CODECVT 1 +#include +#endif + +/** disable deprecations */ +#if defined(__GNUC__) // GCC or clang +#define CLI11_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") +#define CLI11_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") + +#define CLI11_DIAGNOSTIC_IGNORE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") + +#elif defined(_MSC_VER) +#define CLI11_DIAGNOSTIC_PUSH __pragma(warning(push)) +#define CLI11_DIAGNOSTIC_POP __pragma(warning(pop)) + +#define CLI11_DIAGNOSTIC_IGNORE_DEPRECATED __pragma(warning(disable : 4996)) + +#else +#define CLI11_DIAGNOSTIC_PUSH +#define CLI11_DIAGNOSTIC_POP + +#define CLI11_DIAGNOSTIC_IGNORE_DEPRECATED + +#endif + +/** Inline macro **/ +#ifdef CLI11_COMPILE +#define CLI11_INLINE +#else +#define CLI11_INLINE inline +#endif + + + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +#include // NOLINT(build/include) +#else +#include +#include +#endif + + + + +#ifdef CLI11_CPP17 +#include +#endif // CLI11_CPP17 + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +#include +#include // NOLINT(build/include) +#endif // CLI11_HAS_FILESYSTEM + + + +#if defined(_WIN32) +#if !(defined(_AMD64_) || defined(_X86_) || defined(_ARM_)) +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || \ + defined(_M_AMD64) +#define _AMD64_ +#elif defined(i386) || defined(__i386) || defined(__i386__) || defined(__i386__) || defined(_M_IX86) +#define _X86_ +#elif defined(__arm__) || defined(_M_ARM) || defined(_M_ARMT) +#define _ARM_ +#elif defined(__aarch64__) || defined(_M_ARM64) +#define _ARM64_ +#elif defined(_M_ARM64EC) +#define _ARM64EC_ +#endif +#endif + +// first +#ifndef NOMINMAX +// if NOMINMAX is already defined we don't want to mess with that either way +#define NOMINMAX +#include +#undef NOMINMAX +#else +#include +#endif + +// second +#include +// third +#include +#include +#endif + + +namespace CLI { + + +/// Convert a wide string to a narrow string. +CLI11_INLINE std::string narrow(const std::wstring &str); +CLI11_INLINE std::string narrow(const wchar_t *str); +CLI11_INLINE std::string narrow(const wchar_t *str, std::size_t size); + +/// Convert a narrow string to a wide string. +CLI11_INLINE std::wstring widen(const std::string &str); +CLI11_INLINE std::wstring widen(const char *str); +CLI11_INLINE std::wstring widen(const char *str, std::size_t size); + +#ifdef CLI11_CPP17 +CLI11_INLINE std::string narrow(std::wstring_view str); +CLI11_INLINE std::wstring widen(std::string_view str); +#endif // CLI11_CPP17 + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +/// Convert a char-string to a native path correctly. +CLI11_INLINE std::filesystem::path to_path(std::string_view str); +#endif // CLI11_HAS_FILESYSTEM + + + + +namespace detail { + +#if !CLI11_HAS_CODECVT +/// Attempt to set one of the acceptable unicode locales for conversion +CLI11_INLINE void set_unicode_locale() { + static const std::array unicode_locales{{"C.UTF-8", "en_US.UTF-8", ".UTF-8"}}; + + for(const auto &locale_name : unicode_locales) { + if(std::setlocale(LC_ALL, locale_name) != nullptr) { + return; + } + } + throw std::runtime_error("CLI::narrow: could not set locale to C.UTF-8"); +} + +template struct scope_guard_t { + F closure; + + explicit scope_guard_t(F closure_) : closure(closure_) {} + ~scope_guard_t() { closure(); } +}; + +template CLI11_NODISCARD CLI11_INLINE scope_guard_t scope_guard(F &&closure) { + return scope_guard_t{std::forward(closure)}; +} + +#endif // !CLI11_HAS_CODECVT + +CLI11_DIAGNOSTIC_PUSH +CLI11_DIAGNOSTIC_IGNORE_DEPRECATED + +CLI11_INLINE std::string narrow_impl(const wchar_t *str, std::size_t str_size) { +#if CLI11_HAS_CODECVT +#ifdef _WIN32 + return std::wstring_convert>().to_bytes(str, str + str_size); + +#else + return std::wstring_convert>().to_bytes(str, str + str_size); + +#endif // _WIN32 +#else // CLI11_HAS_CODECVT + (void)str_size; + std::mbstate_t state = std::mbstate_t(); + const wchar_t *it = str; + + std::string old_locale = std::setlocale(LC_ALL, nullptr); + auto sg = scope_guard([&] { std::setlocale(LC_ALL, old_locale.c_str()); }); + set_unicode_locale(); + + std::size_t new_size = std::wcsrtombs(nullptr, &it, 0, &state); + if(new_size == static_cast(-1)) { + throw std::runtime_error("CLI::narrow: conversion error in std::wcsrtombs at offset " + + std::to_string(it - str)); + } + std::string result(new_size, '\0'); + std::wcsrtombs(const_cast(result.data()), &str, new_size, &state); + + return result; + +#endif // CLI11_HAS_CODECVT +} + +CLI11_INLINE std::wstring widen_impl(const char *str, std::size_t str_size) { +#if CLI11_HAS_CODECVT +#ifdef _WIN32 + return std::wstring_convert>().from_bytes(str, str + str_size); + +#else + return std::wstring_convert>().from_bytes(str, str + str_size); + +#endif // _WIN32 +#else // CLI11_HAS_CODECVT + (void)str_size; + std::mbstate_t state = std::mbstate_t(); + const char *it = str; + + std::string old_locale = std::setlocale(LC_ALL, nullptr); + auto sg = scope_guard([&] { std::setlocale(LC_ALL, old_locale.c_str()); }); + set_unicode_locale(); + + std::size_t new_size = std::mbsrtowcs(nullptr, &it, 0, &state); + if(new_size == static_cast(-1)) { + throw std::runtime_error("CLI::widen: conversion error in std::mbsrtowcs at offset " + + std::to_string(it - str)); + } + std::wstring result(new_size, L'\0'); + std::mbsrtowcs(const_cast(result.data()), &str, new_size, &state); + + return result; + +#endif // CLI11_HAS_CODECVT +} + +CLI11_DIAGNOSTIC_POP + +} // namespace detail + +CLI11_INLINE std::string narrow(const wchar_t *str, std::size_t str_size) { return detail::narrow_impl(str, str_size); } +CLI11_INLINE std::string narrow(const std::wstring &str) { return detail::narrow_impl(str.data(), str.size()); } +// Flawfinder: ignore +CLI11_INLINE std::string narrow(const wchar_t *str) { return detail::narrow_impl(str, std::wcslen(str)); } + +CLI11_INLINE std::wstring widen(const char *str, std::size_t str_size) { return detail::widen_impl(str, str_size); } +CLI11_INLINE std::wstring widen(const std::string &str) { return detail::widen_impl(str.data(), str.size()); } +// Flawfinder: ignore +CLI11_INLINE std::wstring widen(const char *str) { return detail::widen_impl(str, std::strlen(str)); } + +#ifdef CLI11_CPP17 +CLI11_INLINE std::string narrow(std::wstring_view str) { return detail::narrow_impl(str.data(), str.size()); } +CLI11_INLINE std::wstring widen(std::string_view str) { return detail::widen_impl(str.data(), str.size()); } +#endif // CLI11_CPP17 + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +CLI11_INLINE std::filesystem::path to_path(std::string_view str) { + return std::filesystem::path{ +#ifdef _WIN32 + widen(str) +#else + str +#endif // _WIN32 + }; +} +#endif // CLI11_HAS_FILESYSTEM + + + + +namespace detail { +#ifdef _WIN32 +/// Decode and return UTF-8 argv from GetCommandLineW. +CLI11_INLINE std::vector compute_win32_argv(); +#endif +} // namespace detail + + + +namespace detail { + +#ifdef _WIN32 +CLI11_INLINE std::vector compute_win32_argv() { + std::vector result; + int argc = 0; + + auto deleter = [](wchar_t **ptr) { LocalFree(ptr); }; + // NOLINTBEGIN(*-avoid-c-arrays) + auto wargv = std::unique_ptr(CommandLineToArgvW(GetCommandLineW(), &argc), deleter); + // NOLINTEND(*-avoid-c-arrays) + + if(wargv == nullptr) { + throw std::runtime_error("CommandLineToArgvW failed with code " + std::to_string(GetLastError())); + } + + result.reserve(static_cast(argc)); + for(size_t i = 0; i < static_cast(argc); ++i) { + result.push_back(narrow(wargv[i])); + } + + return result; +} +#endif + +} // namespace detail + + + + +/// Include the items in this namespace to get free conversion of enums to/from streams. +/// (This is available inside CLI as well, so CLI11 will use this without a using statement). +namespace enums { + +/// output streaming for enumerations +template ::value>::type> +std::ostream &operator<<(std::ostream &in, const T &item) { + // make sure this is out of the detail namespace otherwise it won't be found when needed + return in << static_cast::type>(item); +} + +} // namespace enums + +/// Export to CLI namespace +using enums::operator<<; + +namespace detail { +/// a constant defining an expected max vector size defined to be a big number that could be multiplied by 4 and not +/// produce overflow for some expected uses +constexpr int expected_max_vector_size{1 << 29}; +// Based on http://stackoverflow.com/questions/236129/split-a-string-in-c +/// Split a string by a delim +CLI11_INLINE std::vector split(const std::string &s, char delim); + +/// Simple function to join a string +template std::string join(const T &v, std::string delim = ",") { + std::ostringstream s; + auto beg = std::begin(v); + auto end = std::end(v); + if(beg != end) + s << *beg++; + while(beg != end) { + s << delim << *beg++; + } + return s.str(); +} + +/// Simple function to join a string from processed elements +template ::value>::type> +std::string join(const T &v, Callable func, std::string delim = ",") { + std::ostringstream s; + auto beg = std::begin(v); + auto end = std::end(v); + auto loc = s.tellp(); + while(beg != end) { + auto nloc = s.tellp(); + if(nloc > loc) { + s << delim; + loc = nloc; + } + s << func(*beg++); + } + return s.str(); +} + +/// Join a string in reverse order +template std::string rjoin(const T &v, std::string delim = ",") { + std::ostringstream s; + for(std::size_t start = 0; start < v.size(); start++) { + if(start > 0) + s << delim; + s << v[v.size() - start - 1]; + } + return s.str(); +} + +// Based roughly on http://stackoverflow.com/questions/25829143/c-trim-whitespace-from-a-string + +/// Trim whitespace from left of string +CLI11_INLINE std::string <rim(std::string &str); + +/// Trim anything from left of string +CLI11_INLINE std::string <rim(std::string &str, const std::string &filter); + +/// Trim whitespace from right of string +CLI11_INLINE std::string &rtrim(std::string &str); + +/// Trim anything from right of string +CLI11_INLINE std::string &rtrim(std::string &str, const std::string &filter); + +/// Trim whitespace from string +inline std::string &trim(std::string &str) { return ltrim(rtrim(str)); } + +/// Trim anything from string +inline std::string &trim(std::string &str, const std::string filter) { return ltrim(rtrim(str, filter), filter); } + +/// Make a copy of the string and then trim it +inline std::string trim_copy(const std::string &str) { + std::string s = str; + return trim(s); +} + +/// remove quotes at the front and back of a string either '"' or '\'' +CLI11_INLINE std::string &remove_quotes(std::string &str); + +/// remove quotes from all elements of a string vector and process escaped components +CLI11_INLINE void remove_quotes(std::vector &args); + +/// Add a leader to the beginning of all new lines (nothing is added +/// at the start of the first line). `"; "` would be for ini files +/// +/// Can't use Regex, or this would be a subs. +CLI11_INLINE std::string fix_newlines(const std::string &leader, std::string input); + +/// Make a copy of the string and then trim it, any filter string can be used (any char in string is filtered) +inline std::string trim_copy(const std::string &str, const std::string &filter) { + std::string s = str; + return trim(s, filter); +} +/// Print a two part "help" string +CLI11_INLINE std::ostream & +format_help(std::ostream &out, std::string name, const std::string &description, std::size_t wid); + +/// Print subcommand aliases +CLI11_INLINE std::ostream &format_aliases(std::ostream &out, const std::vector &aliases, std::size_t wid); + +/// Verify the first character of an option +/// - is a trigger character, ! has special meaning and new lines would just be annoying to deal with +template bool valid_first_char(T c) { + return ((c != '-') && (static_cast(c) > 33)); // space and '!' not allowed +} + +/// Verify following characters of an option +template bool valid_later_char(T c) { + // = and : are value separators, { has special meaning for option defaults, + // and control codes other than tab would just be annoying to deal with in many places allowing space here has too + // much potential for inadvertent entry errors and bugs + return ((c != '=') && (c != ':') && (c != '{') && ((static_cast(c) > 32) || c == '\t')); +} + +/// Verify an option/subcommand name +CLI11_INLINE bool valid_name_string(const std::string &str); + +/// Verify an app name +inline bool valid_alias_name_string(const std::string &str) { + static const std::string badChars(std::string("\n") + '\0'); + return (str.find_first_of(badChars) == std::string::npos); +} + +/// check if a string is a container segment separator (empty or "%%") +inline bool is_separator(const std::string &str) { + static const std::string sep("%%"); + return (str.empty() || str == sep); +} + +/// Verify that str consists of letters only +inline bool isalpha(const std::string &str) { + return std::all_of(str.begin(), str.end(), [](char c) { return std::isalpha(c, std::locale()); }); +} + +/// Return a lower case version of a string +inline std::string to_lower(std::string str) { + std::transform(std::begin(str), std::end(str), std::begin(str), [](const std::string::value_type &x) { + return std::tolower(x, std::locale()); + }); + return str; +} + +/// remove underscores from a string +inline std::string remove_underscore(std::string str) { + str.erase(std::remove(std::begin(str), std::end(str), '_'), std::end(str)); + return str; +} + +/// Find and replace a substring with another substring +CLI11_INLINE std::string find_and_replace(std::string str, std::string from, std::string to); + +/// check if the flag definitions has possible false flags +inline bool has_default_flag_values(const std::string &flags) { + return (flags.find_first_of("{!") != std::string::npos); +} + +CLI11_INLINE void remove_default_flag_values(std::string &flags); + +/// Check if a string is a member of a list of strings and optionally ignore case or ignore underscores +CLI11_INLINE std::ptrdiff_t find_member(std::string name, + const std::vector names, + bool ignore_case = false, + bool ignore_underscore = false); + +/// Find a trigger string and call a modify callable function that takes the current string and starting position of the +/// trigger and returns the position in the string to search for the next trigger string +template inline std::string find_and_modify(std::string str, std::string trigger, Callable modify) { + std::size_t start_pos = 0; + while((start_pos = str.find(trigger, start_pos)) != std::string::npos) { + start_pos = modify(str, start_pos); + } + return str; +} + +/// close a sequence of characters indicated by a closure character. Brackets allows sub sequences +/// recognized bracket sequences include "'`[(<{ other closure characters are assumed to be literal strings +CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t start, char closure_char); + +/// Split a string '"one two" "three"' into 'one two', 'three' +/// Quote characters can be ` ' or " or bracket characters [{(< with matching to the matching bracket +CLI11_INLINE std::vector split_up(std::string str, char delimiter = '\0'); + +/// get the value of an environmental variable or empty string if empty +CLI11_INLINE std::string get_environment_value(const std::string &env_name); + +/// This function detects an equal or colon followed by an escaped quote after an argument +/// then modifies the string to replace the equality with a space. This is needed +/// to allow the split up function to work properly and is intended to be used with the find_and_modify function +/// the return value is the offset+1 which is required by the find_and_modify function. +CLI11_INLINE std::size_t escape_detect(std::string &str, std::size_t offset); + +/// @brief detect if a string has escapable characters +/// @param str the string to do the detection on +/// @return true if the string has escapable characters +CLI11_INLINE bool has_escapable_character(const std::string &str); + +/// @brief escape all escapable characters +/// @param str the string to escape +/// @return a string with the escapble characters escaped with '\' +CLI11_INLINE std::string add_escaped_characters(const std::string &str); + +/// @brief replace the escaped characters with their equivalent +CLI11_INLINE std::string remove_escaped_characters(const std::string &str); + +/// generate a string with all non printable characters escaped to hex codes +CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape); + +CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string); + +/// extract an escaped binary_string +CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string); + +/// process a quoted string, remove the quotes and if appropriate handle escaped characters +CLI11_INLINE bool process_quoted_string(std::string &str, char string_char = '\"', char literal_char = '\''); + +} // namespace detail + + + + +namespace detail { +CLI11_INLINE std::vector split(const std::string &s, char delim) { + std::vector elems; + // Check to see if empty string, give consistent result + if(s.empty()) { + elems.emplace_back(); + } else { + std::stringstream ss; + ss.str(s); + std::string item; + while(std::getline(ss, item, delim)) { + elems.push_back(item); + } + } + return elems; +} + +CLI11_INLINE std::string <rim(std::string &str) { + auto it = std::find_if(str.begin(), str.end(), [](char ch) { return !std::isspace(ch, std::locale()); }); + str.erase(str.begin(), it); + return str; +} + +CLI11_INLINE std::string <rim(std::string &str, const std::string &filter) { + auto it = std::find_if(str.begin(), str.end(), [&filter](char ch) { return filter.find(ch) == std::string::npos; }); + str.erase(str.begin(), it); + return str; +} + +CLI11_INLINE std::string &rtrim(std::string &str) { + auto it = std::find_if(str.rbegin(), str.rend(), [](char ch) { return !std::isspace(ch, std::locale()); }); + str.erase(it.base(), str.end()); + return str; +} + +CLI11_INLINE std::string &rtrim(std::string &str, const std::string &filter) { + auto it = + std::find_if(str.rbegin(), str.rend(), [&filter](char ch) { return filter.find(ch) == std::string::npos; }); + str.erase(it.base(), str.end()); + return str; +} + +CLI11_INLINE std::string &remove_quotes(std::string &str) { + if(str.length() > 1 && (str.front() == '"' || str.front() == '\'' || str.front() == '`')) { + if(str.front() == str.back()) { + str.pop_back(); + str.erase(str.begin(), str.begin() + 1); + } + } + return str; +} + +CLI11_INLINE std::string &remove_outer(std::string &str, char key) { + if(str.length() > 1 && (str.front() == key)) { + if(str.front() == str.back()) { + str.pop_back(); + str.erase(str.begin(), str.begin() + 1); + } + } + return str; +} + +CLI11_INLINE std::string fix_newlines(const std::string &leader, std::string input) { + std::string::size_type n = 0; + while(n != std::string::npos && n < input.size()) { + n = input.find('\n', n); + if(n != std::string::npos) { + input = input.substr(0, n + 1) + leader + input.substr(n + 1); + n += leader.size(); + } + } + return input; +} + +CLI11_INLINE std::ostream & +format_help(std::ostream &out, std::string name, const std::string &description, std::size_t wid) { + name = " " + name; + out << std::setw(static_cast(wid)) << std::left << name; + if(!description.empty()) { + if(name.length() >= wid) + out << "\n" << std::setw(static_cast(wid)) << ""; + for(const char c : description) { + out.put(c); + if(c == '\n') { + out << std::setw(static_cast(wid)) << ""; + } + } + } + out << "\n"; + return out; +} + +CLI11_INLINE std::ostream &format_aliases(std::ostream &out, const std::vector &aliases, std::size_t wid) { + if(!aliases.empty()) { + out << std::setw(static_cast(wid)) << " aliases: "; + bool front = true; + for(const auto &alias : aliases) { + if(!front) { + out << ", "; + } else { + front = false; + } + out << detail::fix_newlines(" ", alias); + } + out << "\n"; + } + return out; +} + +CLI11_INLINE bool valid_name_string(const std::string &str) { + if(str.empty() || !valid_first_char(str[0])) { + return false; + } + auto e = str.end(); + for(auto c = str.begin() + 1; c != e; ++c) + if(!valid_later_char(*c)) + return false; + return true; +} + +CLI11_INLINE std::string find_and_replace(std::string str, std::string from, std::string to) { + + std::size_t start_pos = 0; + + while((start_pos = str.find(from, start_pos)) != std::string::npos) { + str.replace(start_pos, from.length(), to); + start_pos += to.length(); + } + + return str; +} + +CLI11_INLINE void remove_default_flag_values(std::string &flags) { + auto loc = flags.find_first_of('{', 2); + while(loc != std::string::npos) { + auto finish = flags.find_first_of("},", loc + 1); + if((finish != std::string::npos) && (flags[finish] == '}')) { + flags.erase(flags.begin() + static_cast(loc), + flags.begin() + static_cast(finish) + 1); + } + loc = flags.find_first_of('{', loc + 1); + } + flags.erase(std::remove(flags.begin(), flags.end(), '!'), flags.end()); +} + +CLI11_INLINE std::ptrdiff_t +find_member(std::string name, const std::vector names, bool ignore_case, bool ignore_underscore) { + auto it = std::end(names); + if(ignore_case) { + if(ignore_underscore) { + name = detail::to_lower(detail::remove_underscore(name)); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::to_lower(detail::remove_underscore(local_name)) == name; + }); + } else { + name = detail::to_lower(name); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::to_lower(local_name) == name; + }); + } + + } else if(ignore_underscore) { + name = detail::remove_underscore(name); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::remove_underscore(local_name) == name; + }); + } else { + it = std::find(std::begin(names), std::end(names), name); + } + + return (it != std::end(names)) ? (it - std::begin(names)) : (-1); +} + +static const std::string escapedChars("\b\t\n\f\r\"\\"); +static const std::string escapedCharsCode("btnfr\"\\"); +static const std::string bracketChars{"\"'`[(<{"}; +static const std::string matchBracketChars("\"'`])>}"); + +CLI11_INLINE bool has_escapable_character(const std::string &str) { + return (str.find_first_of(escapedChars) != std::string::npos); +} + +CLI11_INLINE std::string add_escaped_characters(const std::string &str) { + std::string out; + out.reserve(str.size() + 4); + for(char s : str) { + auto sloc = escapedChars.find_first_of(s); + if(sloc != std::string::npos) { + out.push_back('\\'); + out.push_back(escapedCharsCode[sloc]); + } else { + out.push_back(s); + } + } + return out; +} + +CLI11_INLINE std::uint32_t hexConvert(char hc) { + int hcode{0}; + if(hc >= '0' && hc <= '9') { + hcode = (hc - '0'); + } else if(hc >= 'A' && hc <= 'F') { + hcode = (hc - 'A' + 10); + } else if(hc >= 'a' && hc <= 'f') { + hcode = (hc - 'a' + 10); + } else { + hcode = -1; + } + return static_cast(hcode); +} + +CLI11_INLINE char make_char(std::uint32_t code) { return static_cast(static_cast(code)); } + +CLI11_INLINE void append_codepoint(std::string &str, std::uint32_t code) { + if(code < 0x80) { // ascii code equivalent + str.push_back(static_cast(code)); + } else if(code < 0x800) { // \u0080 to \u07FF + // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 + str.push_back(make_char(0xC0 | code >> 6)); + str.push_back(make_char(0x80 | (code & 0x3F))); + } else if(code < 0x10000) { // U+0800...U+FFFF + if(0xD800 <= code && code <= 0xDFFF) { + throw std::invalid_argument("[0xD800, 0xDFFF] are not valid UTF-8."); + } + // 1110yyyy 10yxxxxx 10xxxxxx + str.push_back(make_char(0xE0 | code >> 12)); + str.push_back(make_char(0x80 | (code >> 6 & 0x3F))); + str.push_back(make_char(0x80 | (code & 0x3F))); + } else if(code < 0x110000) { // U+010000 ... U+10FFFF + // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx + str.push_back(make_char(0xF0 | code >> 18)); + str.push_back(make_char(0x80 | (code >> 12 & 0x3F))); + str.push_back(make_char(0x80 | (code >> 6 & 0x3F))); + str.push_back(make_char(0x80 | (code & 0x3F))); + } +} + +CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { + + std::string out; + out.reserve(str.size()); + for(auto loc = str.begin(); loc < str.end(); ++loc) { + if(*loc == '\\') { + if(str.end() - loc < 2) { + throw std::invalid_argument("invalid escape sequence " + str); + } + auto ecloc = escapedCharsCode.find_first_of(*(loc + 1)); + if(ecloc != std::string::npos) { + out.push_back(escapedChars[ecloc]); + ++loc; + } else if(*(loc + 1) == 'u') { + // must have 4 hex characters + if(str.end() - loc < 6) { + throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); + } + std::uint32_t code{0}; + std::uint32_t mplier{16 * 16 * 16}; + for(int ii = 2; ii < 6; ++ii) { + std::uint32_t res = hexConvert(*(loc + ii)); + if(res > 0x0F) { + throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); + } + code += res * mplier; + mplier = mplier / 16; + } + append_codepoint(out, code); + loc += 5; + } else if(*(loc + 1) == 'U') { + // must have 8 hex characters + if(str.end() - loc < 10) { + throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); + } + std::uint32_t code{0}; + std::uint32_t mplier{16 * 16 * 16 * 16 * 16 * 16 * 16}; + for(int ii = 2; ii < 10; ++ii) { + std::uint32_t res = hexConvert(*(loc + ii)); + if(res > 0x0F) { + throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); + } + code += res * mplier; + mplier = mplier / 16; + } + append_codepoint(out, code); + loc += 9; + } else if(*(loc + 1) == '0') { + out.push_back('\0'); + ++loc; + } else { + throw std::invalid_argument(std::string("unrecognized escape sequence \\") + *(loc + 1) + " in " + str); + } + } else { + out.push_back(*loc); + } + } + return out; +} + +CLI11_INLINE std::size_t close_string_quote(const std::string &str, std::size_t start, char closure_char) { + std::size_t loc{0}; + for(loc = start + 1; loc < str.size(); ++loc) { + if(str[loc] == closure_char) { + break; + } + if(str[loc] == '\\') { + // skip the next character for escaped sequences + ++loc; + } + } + return loc; +} + +CLI11_INLINE std::size_t close_literal_quote(const std::string &str, std::size_t start, char closure_char) { + auto loc = str.find_first_of(closure_char, start + 1); + return (loc != std::string::npos ? loc : str.size()); +} + +CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t start, char closure_char) { + + auto bracket_loc = matchBracketChars.find(closure_char); + switch(bracket_loc) { + case 0: + return close_string_quote(str, start, closure_char); + case 1: + case 2: + case std::string::npos: + return close_literal_quote(str, start, closure_char); + default: + break; + } + + std::string closures(1, closure_char); + auto loc = start + 1; + + while(loc < str.size()) { + if(str[loc] == closures.back()) { + closures.pop_back(); + if(closures.empty()) { + return loc; + } + } + bracket_loc = bracketChars.find(str[loc]); + if(bracket_loc != std::string::npos) { + switch(bracket_loc) { + case 0: + loc = close_string_quote(str, loc, str[loc]); + break; + case 1: + case 2: + loc = close_literal_quote(str, loc, str[loc]); + break; + default: + closures.push_back(matchBracketChars[bracket_loc]); + break; + } + } + ++loc; + } + if(loc > str.size()) { + loc = str.size(); + } + return loc; +} + +CLI11_INLINE std::vector split_up(std::string str, char delimiter) { + + auto find_ws = [delimiter](char ch) { + return (delimiter == '\0') ? std::isspace(ch, std::locale()) : (ch == delimiter); + }; + trim(str); + + std::vector output; + while(!str.empty()) { + if(bracketChars.find_first_of(str[0]) != std::string::npos) { + auto bracketLoc = bracketChars.find_first_of(str[0]); + auto end = close_sequence(str, 0, matchBracketChars[bracketLoc]); + if(end >= str.size()) { + output.push_back(std::move(str)); + str.clear(); + } else { + output.push_back(str.substr(0, end + 1)); + if(end + 2 < str.size()) { + str = str.substr(end + 2); + } else { + str.clear(); + } + } + + } else { + auto it = std::find_if(std::begin(str), std::end(str), find_ws); + if(it != std::end(str)) { + std::string value = std::string(str.begin(), it); + output.push_back(value); + str = std::string(it + 1, str.end()); + } else { + output.push_back(str); + str.clear(); + } + } + trim(str); + } + return output; +} + +CLI11_INLINE std::size_t escape_detect(std::string &str, std::size_t offset) { + auto next = str[offset + 1]; + if((next == '\"') || (next == '\'') || (next == '`')) { + auto astart = str.find_last_of("-/ \"\'`", offset - 1); + if(astart != std::string::npos) { + if(str[astart] == ((str[offset] == '=') ? '-' : '/')) + str[offset] = ' '; // interpret this as a space so the split_up works properly + } + } + return offset + 1; +} + +CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape) { + // s is our escaped output string + std::string escaped_string{}; + // loop through all characters + for(char c : string_to_escape) { + // check if a given character is printable + // the cast is necessary to avoid undefined behaviour + if(isprint(static_cast(c)) == 0) { + std::stringstream stream; + // if the character is not printable + // we'll convert it to a hex string using a stringstream + // note that since char is signed we have to cast it to unsigned first + stream << std::hex << static_cast(static_cast(c)); + std::string code = stream.str(); + escaped_string += std::string("\\x") + (code.size() < 2 ? "0" : "") + code; + + } else { + escaped_string.push_back(c); + } + } + if(escaped_string != string_to_escape) { + auto sqLoc = escaped_string.find('\''); + while(sqLoc != std::string::npos) { + escaped_string.replace(sqLoc, sqLoc + 1, "\\x27"); + sqLoc = escaped_string.find('\''); + } + escaped_string.insert(0, "'B\"("); + escaped_string.push_back(')'); + escaped_string.push_back('"'); + escaped_string.push_back('\''); + } + return escaped_string; +} + +CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string) { + size_t ssize = escaped_string.size(); + if(escaped_string.compare(0, 3, "B\"(") == 0 && escaped_string.compare(ssize - 2, 2, ")\"") == 0) { + return true; + } + return (escaped_string.compare(0, 4, "'B\"(") == 0 && escaped_string.compare(ssize - 3, 3, ")\"'") == 0); +} + +CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string) { + std::size_t start{0}; + std::size_t tail{0}; + size_t ssize = escaped_string.size(); + if(escaped_string.compare(0, 3, "B\"(") == 0 && escaped_string.compare(ssize - 2, 2, ")\"") == 0) { + start = 3; + tail = 2; + } else if(escaped_string.compare(0, 4, "'B\"(") == 0 && escaped_string.compare(ssize - 3, 3, ")\"'") == 0) { + start = 4; + tail = 3; + } + + if(start == 0) { + return escaped_string; + } + std::string outstring; + + outstring.reserve(ssize - start - tail); + std::size_t loc = start; + while(loc < ssize - tail) { + // ssize-2 to skip )" at the end + if(escaped_string[loc] == '\\' && (escaped_string[loc + 1] == 'x' || escaped_string[loc + 1] == 'X')) { + auto c1 = escaped_string[loc + 2]; + auto c2 = escaped_string[loc + 3]; + + std::uint32_t res1 = hexConvert(c1); + std::uint32_t res2 = hexConvert(c2); + if(res1 <= 0x0F && res2 <= 0x0F) { + loc += 4; + outstring.push_back(static_cast(res1 * 16 + res2)); + continue; + } + } + outstring.push_back(escaped_string[loc]); + ++loc; + } + return outstring; +} + +CLI11_INLINE void remove_quotes(std::vector &args) { + for(auto &arg : args) { + if(arg.front() == '\"' && arg.back() == '\"') { + remove_quotes(arg); + // only remove escaped for string arguments not literal strings + arg = remove_escaped_characters(arg); + } else { + remove_quotes(arg); + } + } +} + +CLI11_INLINE bool process_quoted_string(std::string &str, char string_char, char literal_char) { + if(str.size() <= 1) { + return false; + } + if(detail::is_binary_escaped_string(str)) { + str = detail::extract_binary_string(str); + return true; + } + if(str.front() == string_char && str.back() == string_char) { + detail::remove_outer(str, string_char); + if(str.find_first_of('\\') != std::string::npos) { + str = detail::remove_escaped_characters(str); + } + return true; + } + if((str.front() == literal_char || str.front() == '`') && str.back() == str.front()) { + detail::remove_outer(str, str.front()); + return true; + } + return false; +} + +std::string get_environment_value(const std::string &env_name) { + char *buffer = nullptr; + std::string ename_string; + +#ifdef _MSC_VER + // Windows version + std::size_t sz = 0; + if(_dupenv_s(&buffer, &sz, env_name.c_str()) == 0 && buffer != nullptr) { + ename_string = std::string(buffer); + free(buffer); + } +#else + // This also works on Windows, but gives a warning + buffer = std::getenv(env_name.c_str()); + if(buffer != nullptr) { + ename_string = std::string(buffer); + } +#endif + return ename_string; +} + +} // namespace detail + + + +// Use one of these on all error classes. +// These are temporary and are undef'd at the end of this file. +#define CLI11_ERROR_DEF(parent, name) \ + protected: \ + name(std::string ename, std::string msg, int exit_code) : parent(std::move(ename), std::move(msg), exit_code) {} \ + name(std::string ename, std::string msg, ExitCodes exit_code) \ + : parent(std::move(ename), std::move(msg), exit_code) {} \ + \ + public: \ + name(std::string msg, ExitCodes exit_code) : parent(#name, std::move(msg), exit_code) {} \ + name(std::string msg, int exit_code) : parent(#name, std::move(msg), exit_code) {} + +// This is added after the one above if a class is used directly and builds its own message +#define CLI11_ERROR_SIMPLE(name) \ + explicit name(std::string msg) : name(#name, msg, ExitCodes::name) {} + +/// These codes are part of every error in CLI. They can be obtained from e using e.exit_code or as a quick shortcut, +/// int values from e.get_error_code(). +enum class ExitCodes { + Success = 0, + IncorrectConstruction = 100, + BadNameString, + OptionAlreadyAdded, + FileError, + ConversionError, + ValidationError, + RequiredError, + RequiresError, + ExcludesError, + ExtrasError, + ConfigError, + InvalidError, + HorribleError, + OptionNotFound, + ArgumentMismatch, + BaseClass = 127 +}; + +// Error definitions + +/// @defgroup error_group Errors +/// @brief Errors thrown by CLI11 +/// +/// These are the errors that can be thrown. Some of them, like CLI::Success, are not really errors. +/// @{ + +/// All errors derive from this one +class Error : public std::runtime_error { + int actual_exit_code; + std::string error_name{"Error"}; + + public: + CLI11_NODISCARD int get_exit_code() const { return actual_exit_code; } + + CLI11_NODISCARD std::string get_name() const { return error_name; } + + Error(std::string name, std::string msg, int exit_code = static_cast(ExitCodes::BaseClass)) + : runtime_error(msg), actual_exit_code(exit_code), error_name(std::move(name)) {} + + Error(std::string name, std::string msg, ExitCodes exit_code) : Error(name, msg, static_cast(exit_code)) {} +}; + +// Note: Using Error::Error constructors does not work on GCC 4.7 + +/// Construction errors (not in parsing) +class ConstructionError : public Error { + CLI11_ERROR_DEF(Error, ConstructionError) +}; + +/// Thrown when an option is set to conflicting values (non-vector and multi args, for example) +class IncorrectConstruction : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, IncorrectConstruction) + CLI11_ERROR_SIMPLE(IncorrectConstruction) + static IncorrectConstruction PositionalFlag(std::string name) { + return IncorrectConstruction(name + ": Flags cannot be positional"); + } + static IncorrectConstruction Set0Opt(std::string name) { + return IncorrectConstruction(name + ": Cannot set 0 expected, use a flag instead"); + } + static IncorrectConstruction SetFlag(std::string name) { + return IncorrectConstruction(name + ": Cannot set an expected number for flags"); + } + static IncorrectConstruction ChangeNotVector(std::string name) { + return IncorrectConstruction(name + ": You can only change the expected arguments for vectors"); + } + static IncorrectConstruction AfterMultiOpt(std::string name) { + return IncorrectConstruction( + name + ": You can't change expected arguments after you've changed the multi option policy!"); + } + static IncorrectConstruction MissingOption(std::string name) { + return IncorrectConstruction("Option " + name + " is not defined"); + } + static IncorrectConstruction MultiOptionPolicy(std::string name) { + return IncorrectConstruction(name + ": multi_option_policy only works for flags and exact value options"); + } +}; + +/// Thrown on construction of a bad name +class BadNameString : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, BadNameString) + CLI11_ERROR_SIMPLE(BadNameString) + static BadNameString OneCharName(std::string name) { return BadNameString("Invalid one char name: " + name); } + static BadNameString MissingDash(std::string name) { + return BadNameString("Long names strings require 2 dashes " + name); + } + static BadNameString BadLongName(std::string name) { return BadNameString("Bad long name: " + name); } + static BadNameString BadPositionalName(std::string name) { + return BadNameString("Invalid positional Name: " + name); + } + static BadNameString DashesOnly(std::string name) { + return BadNameString("Must have a name, not just dashes: " + name); + } + static BadNameString MultiPositionalNames(std::string name) { + return BadNameString("Only one positional name allowed, remove: " + name); + } +}; + +/// Thrown when an option already exists +class OptionAlreadyAdded : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, OptionAlreadyAdded) + explicit OptionAlreadyAdded(std::string name) + : OptionAlreadyAdded(name + " is already added", ExitCodes::OptionAlreadyAdded) {} + static OptionAlreadyAdded Requires(std::string name, std::string other) { + return {name + " requires " + other, ExitCodes::OptionAlreadyAdded}; + } + static OptionAlreadyAdded Excludes(std::string name, std::string other) { + return {name + " excludes " + other, ExitCodes::OptionAlreadyAdded}; + } +}; + +// Parsing errors + +/// Anything that can error in Parse +class ParseError : public Error { + CLI11_ERROR_DEF(Error, ParseError) +}; + +// Not really "errors" + +/// This is a successful completion on parsing, supposed to exit +class Success : public ParseError { + CLI11_ERROR_DEF(ParseError, Success) + Success() : Success("Successfully completed, should be caught and quit", ExitCodes::Success) {} +}; + +/// -h or --help on command line +class CallForHelp : public Success { + CLI11_ERROR_DEF(Success, CallForHelp) + CallForHelp() : CallForHelp("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// Usually something like --help-all on command line +class CallForAllHelp : public Success { + CLI11_ERROR_DEF(Success, CallForAllHelp) + CallForAllHelp() + : CallForAllHelp("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// -v or --version on command line +class CallForVersion : public Success { + CLI11_ERROR_DEF(Success, CallForVersion) + CallForVersion() + : CallForVersion("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// Does not output a diagnostic in CLI11_PARSE, but allows main() to return with a specific error code. +class RuntimeError : public ParseError { + CLI11_ERROR_DEF(ParseError, RuntimeError) + explicit RuntimeError(int exit_code = 1) : RuntimeError("Runtime error", exit_code) {} +}; + +/// Thrown when parsing an INI file and it is missing +class FileError : public ParseError { + CLI11_ERROR_DEF(ParseError, FileError) + CLI11_ERROR_SIMPLE(FileError) + static FileError Missing(std::string name) { return FileError(name + " was not readable (missing?)"); } +}; + +/// Thrown when conversion call back fails, such as when an int fails to coerce to a string +class ConversionError : public ParseError { + CLI11_ERROR_DEF(ParseError, ConversionError) + CLI11_ERROR_SIMPLE(ConversionError) + ConversionError(std::string member, std::string name) + : ConversionError("The value " + member + " is not an allowed value for " + name) {} + ConversionError(std::string name, std::vector results) + : ConversionError("Could not convert: " + name + " = " + detail::join(results)) {} + static ConversionError TooManyInputsFlag(std::string name) { + return ConversionError(name + ": too many inputs for a flag"); + } + static ConversionError TrueFalse(std::string name) { + return ConversionError(name + ": Should be true/false or a number"); + } +}; + +/// Thrown when validation of results fails +class ValidationError : public ParseError { + CLI11_ERROR_DEF(ParseError, ValidationError) + CLI11_ERROR_SIMPLE(ValidationError) + explicit ValidationError(std::string name, std::string msg) : ValidationError(name + ": " + msg) {} +}; + +/// Thrown when a required option is missing +class RequiredError : public ParseError { + CLI11_ERROR_DEF(ParseError, RequiredError) + explicit RequiredError(std::string name) : RequiredError(name + " is required", ExitCodes::RequiredError) {} + static RequiredError Subcommand(std::size_t min_subcom) { + if(min_subcom == 1) { + return RequiredError("A subcommand"); + } + return {"Requires at least " + std::to_string(min_subcom) + " subcommands", ExitCodes::RequiredError}; + } + static RequiredError + Option(std::size_t min_option, std::size_t max_option, std::size_t used, const std::string &option_list) { + if((min_option == 1) && (max_option == 1) && (used == 0)) + return RequiredError("Exactly 1 option from [" + option_list + "]"); + if((min_option == 1) && (max_option == 1) && (used > 1)) { + return {"Exactly 1 option from [" + option_list + "] is required and " + std::to_string(used) + + " were given", + ExitCodes::RequiredError}; + } + if((min_option == 1) && (used == 0)) + return RequiredError("At least 1 option from [" + option_list + "]"); + if(used < min_option) { + return {"Requires at least " + std::to_string(min_option) + " options used and only " + + std::to_string(used) + "were given from [" + option_list + "]", + ExitCodes::RequiredError}; + } + if(max_option == 1) + return {"Requires at most 1 options be given from [" + option_list + "]", ExitCodes::RequiredError}; + + return {"Requires at most " + std::to_string(max_option) + " options be used and " + std::to_string(used) + + "were given from [" + option_list + "]", + ExitCodes::RequiredError}; + } +}; + +/// Thrown when the wrong number of arguments has been received +class ArgumentMismatch : public ParseError { + CLI11_ERROR_DEF(ParseError, ArgumentMismatch) + CLI11_ERROR_SIMPLE(ArgumentMismatch) + ArgumentMismatch(std::string name, int expected, std::size_t received) + : ArgumentMismatch(expected > 0 ? ("Expected exactly " + std::to_string(expected) + " arguments to " + name + + ", got " + std::to_string(received)) + : ("Expected at least " + std::to_string(-expected) + " arguments to " + name + + ", got " + std::to_string(received)), + ExitCodes::ArgumentMismatch) {} + + static ArgumentMismatch AtLeast(std::string name, int num, std::size_t received) { + return ArgumentMismatch(name + ": At least " + std::to_string(num) + " required but received " + + std::to_string(received)); + } + static ArgumentMismatch AtMost(std::string name, int num, std::size_t received) { + return ArgumentMismatch(name + ": At Most " + std::to_string(num) + " required but received " + + std::to_string(received)); + } + static ArgumentMismatch TypedAtLeast(std::string name, int num, std::string type) { + return ArgumentMismatch(name + ": " + std::to_string(num) + " required " + type + " missing"); + } + static ArgumentMismatch FlagOverride(std::string name) { + return ArgumentMismatch(name + " was given a disallowed flag override"); + } + static ArgumentMismatch PartialType(std::string name, int num, std::string type) { + return ArgumentMismatch(name + ": " + type + " only partially specified: " + std::to_string(num) + + " required for each element"); + } +}; + +/// Thrown when a requires option is missing +class RequiresError : public ParseError { + CLI11_ERROR_DEF(ParseError, RequiresError) + RequiresError(std::string curname, std::string subname) + : RequiresError(curname + " requires " + subname, ExitCodes::RequiresError) {} +}; + +/// Thrown when an excludes option is present +class ExcludesError : public ParseError { + CLI11_ERROR_DEF(ParseError, ExcludesError) + ExcludesError(std::string curname, std::string subname) + : ExcludesError(curname + " excludes " + subname, ExitCodes::ExcludesError) {} +}; + +/// Thrown when too many positionals or options are found +class ExtrasError : public ParseError { + CLI11_ERROR_DEF(ParseError, ExtrasError) + explicit ExtrasError(std::vector args) + : ExtrasError((args.size() > 1 ? "The following arguments were not expected: " + : "The following argument was not expected: ") + + detail::rjoin(args, " "), + ExitCodes::ExtrasError) {} + ExtrasError(const std::string &name, std::vector args) + : ExtrasError(name, + (args.size() > 1 ? "The following arguments were not expected: " + : "The following argument was not expected: ") + + detail::rjoin(args, " "), + ExitCodes::ExtrasError) {} +}; + +/// Thrown when extra values are found in an INI file +class ConfigError : public ParseError { + CLI11_ERROR_DEF(ParseError, ConfigError) + CLI11_ERROR_SIMPLE(ConfigError) + static ConfigError Extras(std::string item) { return ConfigError("INI was not able to parse " + item); } + static ConfigError NotConfigurable(std::string item) { + return ConfigError(item + ": This option is not allowed in a configuration file"); + } +}; + +/// Thrown when validation fails before parsing +class InvalidError : public ParseError { + CLI11_ERROR_DEF(ParseError, InvalidError) + explicit InvalidError(std::string name) + : InvalidError(name + ": Too many positional arguments with unlimited expected args", ExitCodes::InvalidError) { + } +}; + +/// This is just a safety check to verify selection and parsing match - you should not ever see it +/// Strings are directly added to this error, but again, it should never be seen. +class HorribleError : public ParseError { + CLI11_ERROR_DEF(ParseError, HorribleError) + CLI11_ERROR_SIMPLE(HorribleError) +}; + +// After parsing + +/// Thrown when counting a non-existent option +class OptionNotFound : public Error { + CLI11_ERROR_DEF(Error, OptionNotFound) + explicit OptionNotFound(std::string name) : OptionNotFound(name + " not found", ExitCodes::OptionNotFound) {} +}; + +#undef CLI11_ERROR_DEF +#undef CLI11_ERROR_SIMPLE + +/// @} + + + + +// Type tools + +// Utilities for type enabling +namespace detail { +// Based generally on https://rmf.io/cxx11/almost-static-if +/// Simple empty scoped class +enum class enabler {}; + +/// An instance to use in EnableIf +constexpr enabler dummy = {}; +} // namespace detail + +/// A copy of enable_if_t from C++14, compatible with C++11. +/// +/// We could check to see if C++14 is being used, but it does not hurt to redefine this +/// (even Google does this: https://github.com/google/skia/blob/main/include/private/SkTLogic.h) +/// It is not in the std namespace anyway, so no harm done. +template using enable_if_t = typename std::enable_if::type; + +/// A copy of std::void_t from C++17 (helper for C++11 and C++14) +template struct make_void { + using type = void; +}; + +/// A copy of std::void_t from C++17 - same reasoning as enable_if_t, it does not hurt to redefine +template using void_t = typename make_void::type; + +/// A copy of std::conditional_t from C++14 - same reasoning as enable_if_t, it does not hurt to redefine +template using conditional_t = typename std::conditional::type; + +/// Check to see if something is bool (fail check by default) +template struct is_bool : std::false_type {}; + +/// Check to see if something is bool (true if actually a bool) +template <> struct is_bool : std::true_type {}; + +/// Check to see if something is a shared pointer +template struct is_shared_ptr : std::false_type {}; + +/// Check to see if something is a shared pointer (True if really a shared pointer) +template struct is_shared_ptr> : std::true_type {}; + +/// Check to see if something is a shared pointer (True if really a shared pointer) +template struct is_shared_ptr> : std::true_type {}; + +/// Check to see if something is copyable pointer +template struct is_copyable_ptr { + static bool const value = is_shared_ptr::value || std::is_pointer::value; +}; + +/// This can be specialized to override the type deduction for IsMember. +template struct IsMemberType { + using type = T; +}; + +/// The main custom type needed here is const char * should be a string. +template <> struct IsMemberType { + using type = std::string; +}; + +namespace detail { + +// These are utilities for IsMember and other transforming objects + +/// Handy helper to access the element_type generically. This is not part of is_copyable_ptr because it requires that +/// pointer_traits be valid. + +/// not a pointer +template struct element_type { + using type = T; +}; + +template struct element_type::value>::type> { + using type = typename std::pointer_traits::element_type; +}; + +/// Combination of the element type and value type - remove pointer (including smart pointers) and get the value_type of +/// the container +template struct element_value_type { + using type = typename element_type::type::value_type; +}; + +/// Adaptor for set-like structure: This just wraps a normal container in a few utilities that do almost nothing. +template struct pair_adaptor : std::false_type { + using value_type = typename T::value_type; + using first_type = typename std::remove_const::type; + using second_type = typename std::remove_const::type; + + /// Get the first value (really just the underlying value) + template static auto first(Q &&pair_value) -> decltype(std::forward(pair_value)) { + return std::forward(pair_value); + } + /// Get the second value (really just the underlying value) + template static auto second(Q &&pair_value) -> decltype(std::forward(pair_value)) { + return std::forward(pair_value); + } +}; + +/// Adaptor for map-like structure (true version, must have key_type and mapped_type). +/// This wraps a mapped container in a few utilities access it in a general way. +template +struct pair_adaptor< + T, + conditional_t, void>> + : std::true_type { + using value_type = typename T::value_type; + using first_type = typename std::remove_const::type; + using second_type = typename std::remove_const::type; + + /// Get the first value (really just the underlying value) + template static auto first(Q &&pair_value) -> decltype(std::get<0>(std::forward(pair_value))) { + return std::get<0>(std::forward(pair_value)); + } + /// Get the second value (really just the underlying value) + template static auto second(Q &&pair_value) -> decltype(std::get<1>(std::forward(pair_value))) { + return std::get<1>(std::forward(pair_value)); + } +}; + +// Warning is suppressed due to "bug" in gcc<5.0 and gcc 7.0 with c++17 enabled that generates a Wnarrowing warning +// in the unevaluated context even if the function that was using this wasn't used. The standard says narrowing in +// brace initialization shouldn't be allowed but for backwards compatibility gcc allows it in some contexts. It is a +// little fuzzy what happens in template constructs and I think that was something GCC took a little while to work out. +// But regardless some versions of gcc generate a warning when they shouldn't from the following code so that should be +// suppressed +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif +// check for constructibility from a specific type and copy assignable used in the parse detection +template class is_direct_constructible { + template + static auto test(int, std::true_type) -> decltype( +// NVCC warns about narrowing conversions here +#ifdef __CUDACC__ +#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__ +#pragma nv_diag_suppress 2361 +#else +#pragma diag_suppress 2361 +#endif +#endif + TT{std::declval()} +#ifdef __CUDACC__ +#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__ +#pragma nv_diag_default 2361 +#else +#pragma diag_default 2361 +#endif +#endif + , + std::is_move_assignable()); + + template static auto test(int, std::false_type) -> std::false_type; + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0, typename std::is_constructible::type()))::value; +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +// Check for output streamability +// Based on https://stackoverflow.com/questions/22758291/how-can-i-detect-if-a-type-can-be-streamed-to-an-stdostream + +template class is_ostreamable { + template + static auto test(int) -> decltype(std::declval() << std::declval(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Check for input streamability +template class is_istreamable { + template + static auto test(int) -> decltype(std::declval() >> std::declval(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Check for complex +template class is_complex { + template + static auto test(int) -> decltype(std::declval().real(), std::declval().imag(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Templated operation to get a value from a stream +template ::value, detail::enabler> = detail::dummy> +bool from_stream(const std::string &istring, T &obj) { + std::istringstream is; + is.str(istring); + is >> obj; + return !is.fail() && !is.rdbuf()->in_avail(); +} + +template ::value, detail::enabler> = detail::dummy> +bool from_stream(const std::string & /*istring*/, T & /*obj*/) { + return false; +} + +// check to see if an object is a mutable container (fail by default) +template struct is_mutable_container : std::false_type {}; + +/// type trait to test if a type is a mutable container meaning it has a value_type, it has an iterator, a clear, and +/// end methods and an insert function. And for our purposes we exclude std::string and types that can be constructed +/// from a std::string +template +struct is_mutable_container< + T, + conditional_t().end()), + decltype(std::declval().clear()), + decltype(std::declval().insert(std::declval().end())>(), + std::declval()))>, + void>> : public conditional_t::value || + std::is_constructible::value, + std::false_type, + std::true_type> {}; + +// check to see if an object is a mutable container (fail by default) +template struct is_readable_container : std::false_type {}; + +/// type trait to test if a type is a container meaning it has a value_type, it has an iterator, a clear, and an end +/// methods and an insert function. And for our purposes we exclude std::string and types that can be constructed from +/// a std::string +template +struct is_readable_container< + T, + conditional_t().end()), decltype(std::declval().begin())>, void>> + : public std::true_type {}; + +// check to see if an object is a wrapper (fail by default) +template struct is_wrapper : std::false_type {}; + +// check if an object is a wrapper (it has a value_type defined) +template +struct is_wrapper, void>> : public std::true_type {}; + +// Check for tuple like types, as in classes with a tuple_size type trait +template class is_tuple_like { + template + // static auto test(int) + // -> decltype(std::conditional<(std::tuple_size::value > 0), std::true_type, std::false_type>::type()); + static auto test(int) -> decltype(std::tuple_size::type>::value, std::true_type{}); + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Convert an object to a string (directly forward if this can become a string) +template ::value, detail::enabler> = detail::dummy> +auto to_string(T &&value) -> decltype(std::forward(value)) { + return std::forward(value); +} + +/// Construct a string from the object +template ::value && !std::is_convertible::value, + detail::enabler> = detail::dummy> +std::string to_string(const T &value) { + return std::string(value); // NOLINT(google-readability-casting) +} + +/// Convert an object to a string (streaming must be supported for that type) +template ::value && !std::is_constructible::value && + is_ostreamable::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&value) { + std::stringstream stream; + stream << value; + return stream.str(); +} + +/// If conversion is not supported, return an empty string (streaming is not supported for that type) +template ::value && !is_ostreamable::value && + !is_readable_container::type>::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&) { + return {}; +} + +/// convert a readable container to a string +template ::value && !is_ostreamable::value && + is_readable_container::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&variable) { + auto cval = variable.begin(); + auto end = variable.end(); + if(cval == end) { + return {"{}"}; + } + std::vector defaults; + while(cval != end) { + defaults.emplace_back(CLI::detail::to_string(*cval)); + ++cval; + } + return {"[" + detail::join(defaults) + "]"}; +} + +/// special template overload +template ::value, detail::enabler> = detail::dummy> +auto checked_to_string(T &&value) -> decltype(to_string(std::forward(value))) { + return to_string(std::forward(value)); +} + +/// special template overload +template ::value, detail::enabler> = detail::dummy> +std::string checked_to_string(T &&) { + return std::string{}; +} +/// get a string as a convertible value for arithmetic types +template ::value, detail::enabler> = detail::dummy> +std::string value_string(const T &value) { + return std::to_string(value); +} +/// get a string as a convertible value for enumerations +template ::value, detail::enabler> = detail::dummy> +std::string value_string(const T &value) { + return std::to_string(static_cast::type>(value)); +} +/// for other types just use the regular to_string function +template ::value && !std::is_arithmetic::value, detail::enabler> = detail::dummy> +auto value_string(const T &value) -> decltype(to_string(value)) { + return to_string(value); +} + +/// template to get the underlying value type if it exists or use a default +template struct wrapped_type { + using type = def; +}; + +/// Type size for regular object types that do not look like a tuple +template struct wrapped_type::value>::type> { + using type = typename T::value_type; +}; + +/// This will only trigger for actual void type +template struct type_count_base { + static const int value{0}; +}; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count_base::value && !is_mutable_container::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; + +/// the base tuple size +template +struct type_count_base::value && !is_mutable_container::value>::type> { + static constexpr int value{std::tuple_size::value}; +}; + +/// Type count base for containers is the type_count_base of the individual element +template struct type_count_base::value>::type> { + static constexpr int value{type_count_base::value}; +}; + +/// Set of overloads to get the type size of an object + +/// forward declare the subtype_count structure +template struct subtype_count; + +/// forward declare the subtype_count_min structure +template struct subtype_count_min; + +/// This will only trigger for actual void type +template struct type_count { + static const int value{0}; +}; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count::value && !is_tuple_like::value && !is_complex::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; + +/// Type size for complex since it sometimes looks like a wrapper +template struct type_count::value>::type> { + static constexpr int value{2}; +}; + +/// Type size of types that are wrappers,except complex and tuples(which can also be wrappers sometimes) +template struct type_count::value>::type> { + static constexpr int value{subtype_count::value}; +}; + +/// Type size of types that are wrappers,except containers complex and tuples(which can also be wrappers sometimes) +template +struct type_count::value && !is_complex::value && !is_tuple_like::value && + !is_mutable_container::value>::type> { + static constexpr int value{type_count::value}; +}; + +/// 0 if the index > tuple size +template +constexpr typename std::enable_if::value, int>::type tuple_type_size() { + return 0; +} + +/// Recursively generate the tuple type name +template + constexpr typename std::enable_if < I::value, int>::type tuple_type_size() { + return subtype_count::type>::value + tuple_type_size(); +} + +/// Get the type size of the sum of type sizes for all the individual tuple types +template struct type_count::value>::type> { + static constexpr int value{tuple_type_size()}; +}; + +/// definition of subtype count +template struct subtype_count { + static constexpr int value{is_mutable_container::value ? expected_max_vector_size : type_count::value}; +}; + +/// This will only trigger for actual void type +template struct type_count_min { + static const int value{0}; +}; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count_min< + T, + typename std::enable_if::value && !is_tuple_like::value && !is_wrapper::value && + !is_complex::value && !std::is_void::value>::type> { + static constexpr int value{type_count::value}; +}; + +/// Type size for complex since it sometimes looks like a wrapper +template struct type_count_min::value>::type> { + static constexpr int value{1}; +}; + +/// Type size min of types that are wrappers,except complex and tuples(which can also be wrappers sometimes) +template +struct type_count_min< + T, + typename std::enable_if::value && !is_complex::value && !is_tuple_like::value>::type> { + static constexpr int value{subtype_count_min::value}; +}; + +/// 0 if the index > tuple size +template +constexpr typename std::enable_if::value, int>::type tuple_type_size_min() { + return 0; +} + +/// Recursively generate the tuple type name +template + constexpr typename std::enable_if < I::value, int>::type tuple_type_size_min() { + return subtype_count_min::type>::value + tuple_type_size_min(); +} + +/// Get the type size of the sum of type sizes for all the individual tuple types +template struct type_count_min::value>::type> { + static constexpr int value{tuple_type_size_min()}; +}; + +/// definition of subtype count +template struct subtype_count_min { + static constexpr int value{is_mutable_container::value + ? ((type_count::value < expected_max_vector_size) ? type_count::value : 0) + : type_count_min::value}; +}; + +/// This will only trigger for actual void type +template struct expected_count { + static const int value{0}; +}; + +/// For most types the number of expected items is 1 +template +struct expected_count::value && !is_wrapper::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; +/// number of expected items in a vector +template struct expected_count::value>::type> { + static constexpr int value{expected_max_vector_size}; +}; + +/// number of expected items in a vector +template +struct expected_count::value && is_wrapper::value>::type> { + static constexpr int value{expected_count::value}; +}; + +// Enumeration of the different supported categorizations of objects +enum class object_category : int { + char_value = 1, + integral_value = 2, + unsigned_integral = 4, + enumeration = 6, + boolean_value = 8, + floating_point = 10, + number_constructible = 12, + double_constructible = 14, + integer_constructible = 16, + // string like types + string_assignable = 23, + string_constructible = 24, + wstring_assignable = 25, + wstring_constructible = 26, + other = 45, + // special wrapper or container types + wrapper_value = 50, + complex_number = 60, + tuple_value = 70, + container_value = 80, + +}; + +/// Set of overloads to classify an object according to type + +/// some type that is not otherwise recognized +template struct classify_object { + static constexpr object_category value{object_category::other}; +}; + +/// Signed integers +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_same::value && std::is_signed::value && + !is_bool::value && !std::is_enum::value>::type> { + static constexpr object_category value{object_category::integral_value}; +}; + +/// Unsigned integers +template +struct classify_object::value && std::is_unsigned::value && + !std::is_same::value && !is_bool::value>::type> { + static constexpr object_category value{object_category::unsigned_integral}; +}; + +/// single character values +template +struct classify_object::value && !std::is_enum::value>::type> { + static constexpr object_category value{object_category::char_value}; +}; + +/// Boolean values +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::boolean_value}; +}; + +/// Floats +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::floating_point}; +}; +#if defined _MSC_VER +// in MSVC wstring should take precedence if available this isn't as useful on other compilers due to the broader use of +// utf-8 encoding +#define WIDE_STRING_CHECK \ + !std::is_assignable::value && !std::is_constructible::value +#define STRING_CHECK true +#else +#define WIDE_STRING_CHECK true +#define STRING_CHECK !std::is_assignable::value && !std::is_constructible::value +#endif + +/// String and similar direct assignment +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && WIDE_STRING_CHECK && + std::is_assignable::value>::type> { + static constexpr object_category value{object_category::string_assignable}; +}; + +/// String and similar constructible and copy assignment +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && + !std::is_assignable::value && (type_count::value == 1) && + WIDE_STRING_CHECK && std::is_constructible::value>::type> { + static constexpr object_category value{object_category::string_constructible}; +}; + +/// Wide strings +template +struct classify_object::value && !std::is_integral::value && + STRING_CHECK && std::is_assignable::value>::type> { + static constexpr object_category value{object_category::wstring_assignable}; +}; + +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && + !std::is_assignable::value && (type_count::value == 1) && + STRING_CHECK && std::is_constructible::value>::type> { + static constexpr object_category value{object_category::wstring_constructible}; +}; + +/// Enumerations +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::enumeration}; +}; + +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::complex_number}; +}; + +/// Handy helper to contain a bunch of checks that rule out many common types (integers, string like, floating point, +/// vectors, and enumerations +template struct uncommon_type { + using type = typename std::conditional< + !std::is_floating_point::value && !std::is_integral::value && + !std::is_assignable::value && !std::is_constructible::value && + !std::is_assignable::value && !std::is_constructible::value && + !is_complex::value && !is_mutable_container::value && !std::is_enum::value, + std::true_type, + std::false_type>::type; + static constexpr bool value = type::value; +}; + +/// wrapper type +template +struct classify_object::value && is_wrapper::value && + !is_tuple_like::value && uncommon_type::value)>::type> { + static constexpr object_category value{object_category::wrapper_value}; +}; + +/// Assignable from double or int +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && is_direct_constructible::value && + is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::number_constructible}; +}; + +/// Assignable from int +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && !is_direct_constructible::value && + is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::integer_constructible}; +}; + +/// Assignable from double +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && is_direct_constructible::value && + !is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::double_constructible}; +}; + +/// Tuple type +template +struct classify_object< + T, + typename std::enable_if::value && + ((type_count::value >= 2 && !is_wrapper::value) || + (uncommon_type::value && !is_direct_constructible::value && + !is_direct_constructible::value) || + (uncommon_type::value && type_count::value >= 2))>::type> { + static constexpr object_category value{object_category::tuple_value}; + // the condition on this class requires it be like a tuple, but on some compilers (like Xcode) tuples can be + // constructed from just the first element so tuples of can be constructed from a string, which + // could lead to issues so there are two variants of the condition, the first isolates things with a type size >=2 + // mainly to get tuples on Xcode with the exception of wrappers, the second is the main one and just separating out + // those cases that are caught by other object classifications +}; + +/// container type +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::container_value}; +}; + +// Type name print + +/// Was going to be based on +/// http://stackoverflow.com/questions/1055452/c-get-name-of-type-in-template +/// But this is cleaner and works better in this case + +template ::value == object_category::char_value, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "CHAR"; +} + +template ::value == object_category::integral_value || + classify_object::value == object_category::integer_constructible, + detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "INT"; +} + +template ::value == object_category::unsigned_integral, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "UINT"; +} + +template ::value == object_category::floating_point || + classify_object::value == object_category::number_constructible || + classify_object::value == object_category::double_constructible, + detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "FLOAT"; +} + +/// Print name for enumeration types +template ::value == object_category::enumeration, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "ENUM"; +} + +/// Print name for enumeration types +template ::value == object_category::boolean_value, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "BOOLEAN"; +} + +/// Print name for enumeration types +template ::value == object_category::complex_number, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "COMPLEX"; +} + +/// Print for all other types +template ::value >= object_category::string_assignable && + classify_object::value <= object_category::other, + detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "TEXT"; +} +/// typename for tuple value +template ::value == object_category::tuple_value && type_count_base::value >= 2, + detail::enabler> = detail::dummy> +std::string type_name(); // forward declaration + +/// Generate type name for a wrapper or container value +template ::value == object_category::container_value || + classify_object::value == object_category::wrapper_value, + detail::enabler> = detail::dummy> +std::string type_name(); // forward declaration + +/// Print name for single element tuple types +template ::value == object_category::tuple_value && type_count_base::value == 1, + detail::enabler> = detail::dummy> +inline std::string type_name() { + return type_name::type>::type>(); +} + +/// Empty string if the index > tuple size +template +inline typename std::enable_if::value, std::string>::type tuple_name() { + return std::string{}; +} + +/// Recursively generate the tuple type name +template +inline typename std::enable_if<(I < type_count_base::value), std::string>::type tuple_name() { + auto str = std::string{type_name::type>::type>()} + ',' + + tuple_name(); + if(str.back() == ',') + str.pop_back(); + return str; +} + +/// Print type name for tuples with 2 or more elements +template ::value == object_category::tuple_value && type_count_base::value >= 2, + detail::enabler>> +inline std::string type_name() { + auto tname = std::string(1, '[') + tuple_name(); + tname.push_back(']'); + return tname; +} + +/// get the type name for a type that has a value_type member +template ::value == object_category::container_value || + classify_object::value == object_category::wrapper_value, + detail::enabler>> +inline std::string type_name() { + return type_name(); +} + +// Lexical cast + +/// Convert to an unsigned integral +template ::value, detail::enabler> = detail::dummy> +bool integral_conversion(const std::string &input, T &output) noexcept { + if(input.empty() || input.front() == '-') { + return false; + } + char *val{nullptr}; + errno = 0; + std::uint64_t output_ll = std::strtoull(input.c_str(), &val, 0); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + if(val == (input.c_str() + input.size()) && static_cast(output) == output_ll) { + return true; + } + val = nullptr; + std::int64_t output_sll = std::strtoll(input.c_str(), &val, 0); + if(val == (input.c_str() + input.size())) { + output = (output_sll < 0) ? static_cast(0) : static_cast(output_sll); + return (static_cast(output) == output_sll); + } + // remove separators + if(input.find_first_of("_'") != std::string::npos) { + std::string nstring = input; + nstring.erase(std::remove(nstring.begin(), nstring.end(), '_'), nstring.end()); + nstring.erase(std::remove(nstring.begin(), nstring.end(), '\''), nstring.end()); + return integral_conversion(nstring, output); + } + if(input.compare(0, 2, "0o") == 0) { + val = nullptr; + errno = 0; + output_ll = std::strtoull(input.c_str() + 2, &val, 8); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + return (val == (input.c_str() + input.size()) && static_cast(output) == output_ll); + } + if(input.compare(0, 2, "0b") == 0) { + val = nullptr; + errno = 0; + output_ll = std::strtoull(input.c_str() + 2, &val, 2); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + return (val == (input.c_str() + input.size()) && static_cast(output) == output_ll); + } + return false; +} + +/// Convert to a signed integral +template ::value, detail::enabler> = detail::dummy> +bool integral_conversion(const std::string &input, T &output) noexcept { + if(input.empty()) { + return false; + } + char *val = nullptr; + errno = 0; + std::int64_t output_ll = std::strtoll(input.c_str(), &val, 0); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + if(val == (input.c_str() + input.size()) && static_cast(output) == output_ll) { + return true; + } + if(input == "true") { + // this is to deal with a few oddities with flags and wrapper int types + output = static_cast(1); + return true; + } + // remove separators + if(input.find_first_of("_'") != std::string::npos) { + std::string nstring = input; + nstring.erase(std::remove(nstring.begin(), nstring.end(), '_'), nstring.end()); + nstring.erase(std::remove(nstring.begin(), nstring.end(), '\''), nstring.end()); + return integral_conversion(nstring, output); + } + if(input.compare(0, 2, "0o") == 0) { + val = nullptr; + errno = 0; + output_ll = std::strtoll(input.c_str() + 2, &val, 8); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + return (val == (input.c_str() + input.size()) && static_cast(output) == output_ll); + } + if(input.compare(0, 2, "0b") == 0) { + val = nullptr; + errno = 0; + output_ll = std::strtoll(input.c_str() + 2, &val, 2); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + return (val == (input.c_str() + input.size()) && static_cast(output) == output_ll); + } + return false; +} + +/// Convert a flag into an integer value typically binary flags sets errno to nonzero if conversion failed +inline std::int64_t to_flag_value(std::string val) noexcept { + static const std::string trueString("true"); + static const std::string falseString("false"); + if(val == trueString) { + return 1; + } + if(val == falseString) { + return -1; + } + val = detail::to_lower(val); + std::int64_t ret = 0; + if(val.size() == 1) { + if(val[0] >= '1' && val[0] <= '9') { + return (static_cast(val[0]) - '0'); + } + switch(val[0]) { + case '0': + case 'f': + case 'n': + case '-': + ret = -1; + break; + case 't': + case 'y': + case '+': + ret = 1; + break; + default: + errno = EINVAL; + return -1; + } + return ret; + } + if(val == trueString || val == "on" || val == "yes" || val == "enable") { + ret = 1; + } else if(val == falseString || val == "off" || val == "no" || val == "disable") { + ret = -1; + } else { + char *loc_ptr{nullptr}; + ret = std::strtoll(val.c_str(), &loc_ptr, 0); + if(loc_ptr != (val.c_str() + val.size()) && errno == 0) { + errno = EINVAL; + } + } + return ret; +} + +/// Integer conversion +template ::value == object_category::integral_value || + classify_object::value == object_category::unsigned_integral, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + return integral_conversion(input, output); +} + +/// char values +template ::value == object_category::char_value, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + if(input.size() == 1) { + output = static_cast(input[0]); + return true; + } + return integral_conversion(input, output); +} + +/// Boolean values +template ::value == object_category::boolean_value, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + errno = 0; + auto out = to_flag_value(input); + if(errno == 0) { + output = (out > 0); + } else if(errno == ERANGE) { + output = (input[0] != '-'); + } else { + return false; + } + return true; +} + +/// Floats +template ::value == object_category::floating_point, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + if(input.empty()) { + return false; + } + char *val = nullptr; + auto output_ld = std::strtold(input.c_str(), &val); + output = static_cast(output_ld); + if(val == (input.c_str() + input.size())) { + return true; + } + // remove separators + if(input.find_first_of("_'") != std::string::npos) { + std::string nstring = input; + nstring.erase(std::remove(nstring.begin(), nstring.end(), '_'), nstring.end()); + nstring.erase(std::remove(nstring.begin(), nstring.end(), '\''), nstring.end()); + return lexical_cast(nstring, output); + } + return false; +} + +/// complex +template ::value == object_category::complex_number, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + using XC = typename wrapped_type::type; + XC x{0.0}, y{0.0}; + auto str1 = input; + bool worked = false; + auto nloc = str1.find_last_of("+-"); + if(nloc != std::string::npos && nloc > 0) { + worked = lexical_cast(str1.substr(0, nloc), x); + str1 = str1.substr(nloc); + if(str1.back() == 'i' || str1.back() == 'j') + str1.pop_back(); + worked = worked && lexical_cast(str1, y); + } else { + if(str1.back() == 'i' || str1.back() == 'j') { + str1.pop_back(); + worked = lexical_cast(str1, y); + x = XC{0}; + } else { + worked = lexical_cast(str1, x); + y = XC{0}; + } + } + if(worked) { + output = T{x, y}; + return worked; + } + return from_stream(input, output); +} + +/// String and similar direct assignment +template ::value == object_category::string_assignable, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = input; + return true; +} + +/// String and similar constructible and copy assignment +template < + typename T, + enable_if_t::value == object_category::string_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = T(input); + return true; +} + +/// Wide strings +template < + typename T, + enable_if_t::value == object_category::wstring_assignable, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = widen(input); + return true; +} + +template < + typename T, + enable_if_t::value == object_category::wstring_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = T{widen(input)}; + return true; +} + +/// Enumerations +template ::value == object_category::enumeration, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + typename std::underlying_type::type val; + if(!integral_conversion(input, val)) { + return false; + } + output = static_cast(val); + return true; +} + +/// wrapper types +template ::value == object_category::wrapper_value && + std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + typename T::value_type val; + if(lexical_cast(input, val)) { + output = val; + return true; + } + return from_stream(input, output); +} + +template ::value == object_category::wrapper_value && + !std::is_assignable::value && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + typename T::value_type val; + if(lexical_cast(input, val)) { + output = T{val}; + return true; + } + return from_stream(input, output); +} + +/// Assignable from double or int +template < + typename T, + enable_if_t::value == object_category::number_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val = 0; + if(integral_conversion(input, val)) { + output = T(val); + return true; + } + + double dval = 0.0; + if(lexical_cast(input, dval)) { + output = T{dval}; + return true; + } + + return from_stream(input, output); +} + +/// Assignable from int +template < + typename T, + enable_if_t::value == object_category::integer_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val = 0; + if(integral_conversion(input, val)) { + output = T(val); + return true; + } + return from_stream(input, output); +} + +/// Assignable from double +template < + typename T, + enable_if_t::value == object_category::double_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + double val = 0.0; + if(lexical_cast(input, val)) { + output = T{val}; + return true; + } + return from_stream(input, output); +} + +/// Non-string convertible from an int +template ::value == object_category::other && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val = 0; + if(integral_conversion(input, val)) { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4800) +#endif + // with Atomic this could produce a warning due to the conversion but if atomic gets here it is an old style + // so will most likely still work + output = val; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + return true; + } + // LCOV_EXCL_START + // This version of cast is only used for odd cases in an older compilers the fail over + // from_stream is tested elsewhere an not relevant for coverage here + return from_stream(input, output); + // LCOV_EXCL_STOP +} + +/// Non-string parsable by a stream +template ::value == object_category::other && !std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + static_assert(is_istreamable::value, + "option object type must have a lexical cast overload or streaming input operator(>>) defined, if it " + "is convertible from another type use the add_option(...) with XC being the known type"); + return from_stream(input, output); +} + +/// Assign a value through lexical cast operations +/// Strings can be empty so we need to do a little different +template ::value && + (classify_object::value == object_category::string_assignable || + classify_object::value == object_category::string_constructible || + classify_object::value == object_category::wstring_assignable || + classify_object::value == object_category::wstring_constructible), + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations +template ::value && std::is_assignable::value && + classify_object::value != object_category::string_assignable && + classify_object::value != object_category::string_constructible && + classify_object::value != object_category::wstring_assignable && + classify_object::value != object_category::wstring_constructible, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + output = AssignTo{}; + return true; + } + + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations +template ::value && !std::is_assignable::value && + classify_object::value == object_category::wrapper_value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + typename AssignTo::value_type emptyVal{}; + output = emptyVal; + return true; + } + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations for int compatible values +/// mainly for atomic operations on some compilers +template ::value && !std::is_assignable::value && + classify_object::value != object_category::wrapper_value && + std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + output = 0; + return true; + } + int val{0}; + if(lexical_cast(input, val)) { +#if defined(__clang__) +/* on some older clang compilers */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wsign-conversion" +#endif + output = val; +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + return true; + } + return false; +} + +/// Assign a value converted from a string in lexical cast to the output value directly +template ::value && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + ConvertTo val{}; + bool parse_result = (!input.empty()) ? lexical_cast(input, val) : true; + if(parse_result) { + output = val; + } + return parse_result; +} + +/// Assign a value from a lexical cast through constructing a value and move assigning it +template < + typename AssignTo, + typename ConvertTo, + enable_if_t::value && !std::is_assignable::value && + std::is_move_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + ConvertTo val{}; + bool parse_result = input.empty() ? true : lexical_cast(input, val); + if(parse_result) { + output = AssignTo(val); // use () form of constructor to allow some implicit conversions + } + return parse_result; +} + +/// primary lexical conversion operation, 1 string to 1 type of some kind +template ::value <= object_category::other && + classify_object::value <= object_category::wrapper_value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + return lexical_assign(strings[0], output); +} + +/// Lexical conversion if there is only one element but the conversion type is for two, then call a two element +/// constructor +template ::value <= 2) && expected_count::value == 1 && + is_tuple_like::value && type_count_base::value == 2, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + // the remove const is to handle pair types coming from a container + using FirstType = typename std::remove_const::type>::type; + using SecondType = typename std::tuple_element<1, ConvertTo>::type; + FirstType v1; + SecondType v2; + bool retval = lexical_assign(strings[0], v1); + retval = retval && lexical_assign((strings.size() > 1) ? strings[1] : std::string{}, v2); + if(retval) { + output = AssignTo{v1, v2}; + } + return retval; +} + +/// Lexical conversion of a container types of single elements +template ::value && is_mutable_container::value && + type_count::value == 1, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + output.erase(output.begin(), output.end()); + if(strings.empty()) { + return true; + } + if(strings.size() == 1 && strings[0] == "{}") { + return true; + } + bool skip_remaining = false; + if(strings.size() == 2 && strings[0] == "{}" && is_separator(strings[1])) { + skip_remaining = true; + } + for(const auto &elem : strings) { + typename AssignTo::value_type out; + bool retval = lexical_assign(elem, out); + if(!retval) { + return false; + } + output.insert(output.end(), std::move(out)); + if(skip_remaining) { + break; + } + } + return (!output.empty()); +} + +/// Lexical conversion for complex types +template ::value, detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + + if(strings.size() >= 2 && !strings[1].empty()) { + using XC2 = typename wrapped_type::type; + XC2 x{0.0}, y{0.0}; + auto str1 = strings[1]; + if(str1.back() == 'i' || str1.back() == 'j') { + str1.pop_back(); + } + auto worked = lexical_cast(strings[0], x) && lexical_cast(str1, y); + if(worked) { + output = ConvertTo{x, y}; + } + return worked; + } + return lexical_assign(strings[0], output); +} + +/// Conversion to a vector type using a particular single type as the conversion type +template ::value && (expected_count::value == 1) && + (type_count::value == 1), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + bool retval = true; + output.clear(); + output.reserve(strings.size()); + for(const auto &elem : strings) { + + output.emplace_back(); + retval = retval && lexical_assign(elem, output.back()); + } + return (!output.empty()) && retval; +} + +// forward declaration + +/// Lexical conversion of a container types with conversion type of two elements +template ::value && is_mutable_container::value && + type_count_base::value == 2, + detail::enabler> = detail::dummy> +bool lexical_conversion(std::vector strings, AssignTo &output); + +/// Lexical conversion of a vector types with type_size >2 forward declaration +template ::value && is_mutable_container::value && + type_count_base::value != 2 && + ((type_count::value > 2) || + (type_count::value > type_count_base::value)), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output); + +/// Conversion for tuples +template ::value && is_tuple_like::value && + (type_count_base::value != type_count::value || + type_count::value > 2), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output); // forward declaration + +/// Conversion for operations where the assigned type is some class but the conversion is a mutable container or large +/// tuple +template ::value && !is_mutable_container::value && + classify_object::value != object_category::wrapper_value && + (is_mutable_container::value || type_count::value > 2), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + + if(strings.size() > 1 || (!strings.empty() && !(strings.front().empty()))) { + ConvertTo val; + auto retval = lexical_conversion(strings, val); + output = AssignTo{val}; + return retval; + } + output = AssignTo{}; + return true; +} + +/// function template for converting tuples if the static Index is greater than the tuple size +template +inline typename std::enable_if<(I >= type_count_base::value), bool>::type +tuple_conversion(const std::vector &, AssignTo &) { + return true; +} + +/// Conversion of a tuple element where the type size ==1 and not a mutable container +template +inline typename std::enable_if::value && type_count::value == 1, bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + auto retval = lexical_assign(strings[0], output); + strings.erase(strings.begin()); + return retval; +} + +/// Conversion of a tuple element where the type size !=1 but the size is fixed and not a mutable container +template +inline typename std::enable_if::value && (type_count::value > 1) && + type_count::value == type_count_min::value, + bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + auto retval = lexical_conversion(strings, output); + strings.erase(strings.begin(), strings.begin() + type_count::value); + return retval; +} + +/// Conversion of a tuple element where the type is a mutable container or a type with different min and max type sizes +template +inline typename std::enable_if::value || + type_count::value != type_count_min::value, + bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + + std::size_t index{subtype_count_min::value}; + const std::size_t mx_count{subtype_count::value}; + const std::size_t mx{(std::min)(mx_count, strings.size() - 1)}; + + while(index < mx) { + if(is_separator(strings[index])) { + break; + } + ++index; + } + bool retval = lexical_conversion( + std::vector(strings.begin(), strings.begin() + static_cast(index)), output); + if(strings.size() > index) { + strings.erase(strings.begin(), strings.begin() + static_cast(index) + 1); + } else { + strings.clear(); + } + return retval; +} + +/// Tuple conversion operation +template +inline typename std::enable_if<(I < type_count_base::value), bool>::type +tuple_conversion(std::vector strings, AssignTo &output) { + bool retval = true; + using ConvertToElement = typename std:: + conditional::value, typename std::tuple_element::type, ConvertTo>::type; + if(!strings.empty()) { + retval = retval && tuple_type_conversion::type, ConvertToElement>( + strings, std::get(output)); + } + retval = retval && tuple_conversion(std::move(strings), output); + return retval; +} + +/// Lexical conversion of a container types with tuple elements of size 2 +template ::value && is_mutable_container::value && + type_count_base::value == 2, + detail::enabler>> +bool lexical_conversion(std::vector strings, AssignTo &output) { + output.clear(); + while(!strings.empty()) { + + typename std::remove_const::type>::type v1; + typename std::tuple_element<1, typename ConvertTo::value_type>::type v2; + bool retval = tuple_type_conversion(strings, v1); + if(!strings.empty()) { + retval = retval && tuple_type_conversion(strings, v2); + } + if(retval) { + output.insert(output.end(), typename AssignTo::value_type{v1, v2}); + } else { + return false; + } + } + return (!output.empty()); +} + +/// lexical conversion of tuples with type count>2 or tuples of types of some element with a type size>=2 +template ::value && is_tuple_like::value && + (type_count_base::value != type_count::value || + type_count::value > 2), + detail::enabler>> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + static_assert( + !is_tuple_like::value || type_count_base::value == type_count_base::value, + "if the conversion type is defined as a tuple it must be the same size as the type you are converting to"); + return tuple_conversion(strings, output); +} + +/// Lexical conversion of a vector types for everything but tuples of two elements and types of size 1 +template ::value && is_mutable_container::value && + type_count_base::value != 2 && + ((type_count::value > 2) || + (type_count::value > type_count_base::value)), + detail::enabler>> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + bool retval = true; + output.clear(); + std::vector temp; + std::size_t ii{0}; + std::size_t icount{0}; + std::size_t xcm{type_count::value}; + auto ii_max = strings.size(); + while(ii < ii_max) { + temp.push_back(strings[ii]); + ++ii; + ++icount; + if(icount == xcm || is_separator(temp.back()) || ii == ii_max) { + if(static_cast(xcm) > type_count_min::value && is_separator(temp.back())) { + temp.pop_back(); + } + typename AssignTo::value_type temp_out; + retval = retval && + lexical_conversion(temp, temp_out); + temp.clear(); + if(!retval) { + return false; + } + output.insert(output.end(), std::move(temp_out)); + icount = 0; + } + } + return retval; +} + +/// conversion for wrapper types +template ::value == object_category::wrapper_value && + std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + if(strings.empty() || strings.front().empty()) { + output = ConvertTo{}; + return true; + } + typename ConvertTo::value_type val; + if(lexical_conversion(strings, val)) { + output = ConvertTo{val}; + return true; + } + return false; +} + +/// conversion for wrapper types +template ::value == object_category::wrapper_value && + !std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + using ConvertType = typename ConvertTo::value_type; + if(strings.empty() || strings.front().empty()) { + output = ConvertType{}; + return true; + } + ConvertType val; + if(lexical_conversion(strings, val)) { + output = val; + return true; + } + return false; +} + +/// Sum a vector of strings +inline std::string sum_string_vector(const std::vector &values) { + double val{0.0}; + bool fail{false}; + std::string output; + for(const auto &arg : values) { + double tv{0.0}; + auto comp = lexical_cast(arg, tv); + if(!comp) { + errno = 0; + auto fv = detail::to_flag_value(arg); + fail = (errno != 0); + if(fail) { + break; + } + tv = static_cast(fv); + } + val += tv; + } + if(fail) { + for(const auto &arg : values) { + output.append(arg); + } + } else { + std::ostringstream out; + out.precision(16); + out << val; + output = out.str(); + } + return output; +} + +} // namespace detail + + + +namespace detail { + +// Returns false if not a short option. Otherwise, sets opt name and rest and returns true +CLI11_INLINE bool split_short(const std::string ¤t, std::string &name, std::string &rest); + +// Returns false if not a long option. Otherwise, sets opt name and other side of = and returns true +CLI11_INLINE bool split_long(const std::string ¤t, std::string &name, std::string &value); + +// Returns false if not a windows style option. Otherwise, sets opt name and value and returns true +CLI11_INLINE bool split_windows_style(const std::string ¤t, std::string &name, std::string &value); + +// Splits a string into multiple long and short names +CLI11_INLINE std::vector split_names(std::string current); + +/// extract default flag values either {def} or starting with a ! +CLI11_INLINE std::vector> get_default_flag_values(const std::string &str); + +/// Get a vector of short names, one of long names, and a single name +CLI11_INLINE std::tuple, std::vector, std::string> +get_names(const std::vector &input); + +} // namespace detail + + + +namespace detail { + +CLI11_INLINE bool split_short(const std::string ¤t, std::string &name, std::string &rest) { + if(current.size() > 1 && current[0] == '-' && valid_first_char(current[1])) { + name = current.substr(1, 1); + rest = current.substr(2); + return true; + } + return false; +} + +CLI11_INLINE bool split_long(const std::string ¤t, std::string &name, std::string &value) { + if(current.size() > 2 && current.compare(0, 2, "--") == 0 && valid_first_char(current[2])) { + auto loc = current.find_first_of('='); + if(loc != std::string::npos) { + name = current.substr(2, loc - 2); + value = current.substr(loc + 1); + } else { + name = current.substr(2); + value = ""; + } + return true; + } + return false; +} + +CLI11_INLINE bool split_windows_style(const std::string ¤t, std::string &name, std::string &value) { + if(current.size() > 1 && current[0] == '/' && valid_first_char(current[1])) { + auto loc = current.find_first_of(':'); + if(loc != std::string::npos) { + name = current.substr(1, loc - 1); + value = current.substr(loc + 1); + } else { + name = current.substr(1); + value = ""; + } + return true; + } + return false; +} + +CLI11_INLINE std::vector split_names(std::string current) { + std::vector output; + std::size_t val = 0; + while((val = current.find(',')) != std::string::npos) { + output.push_back(trim_copy(current.substr(0, val))); + current = current.substr(val + 1); + } + output.push_back(trim_copy(current)); + return output; +} + +CLI11_INLINE std::vector> get_default_flag_values(const std::string &str) { + std::vector flags = split_names(str); + flags.erase(std::remove_if(flags.begin(), + flags.end(), + [](const std::string &name) { + return ((name.empty()) || (!(((name.find_first_of('{') != std::string::npos) && + (name.back() == '}')) || + (name[0] == '!')))); + }), + flags.end()); + std::vector> output; + output.reserve(flags.size()); + for(auto &flag : flags) { + auto def_start = flag.find_first_of('{'); + std::string defval = "false"; + if((def_start != std::string::npos) && (flag.back() == '}')) { + defval = flag.substr(def_start + 1); + defval.pop_back(); + flag.erase(def_start, std::string::npos); // NOLINT(readability-suspicious-call-argument) + } + flag.erase(0, flag.find_first_not_of("-!")); + output.emplace_back(flag, defval); + } + return output; +} + +CLI11_INLINE std::tuple, std::vector, std::string> +get_names(const std::vector &input) { + + std::vector short_names; + std::vector long_names; + std::string pos_name; + for(std::string name : input) { + if(name.length() == 0) { + continue; + } + if(name.length() > 1 && name[0] == '-' && name[1] != '-') { + if(name.length() == 2 && valid_first_char(name[1])) + short_names.emplace_back(1, name[1]); + else if(name.length() > 2) + throw BadNameString::MissingDash(name); + else + throw BadNameString::OneCharName(name); + } else if(name.length() > 2 && name.substr(0, 2) == "--") { + name = name.substr(2); + if(valid_name_string(name)) + long_names.push_back(name); + else + throw BadNameString::BadLongName(name); + } else if(name == "-" || name == "--") { + throw BadNameString::DashesOnly(name); + } else { + if(!pos_name.empty()) + throw BadNameString::MultiPositionalNames(name); + if(valid_name_string(name)) { + pos_name = name; + } else { + throw BadNameString::BadPositionalName(name); + } + } + } + return std::make_tuple(short_names, long_names, pos_name); +} + +} // namespace detail + + + +class App; + +/// Holds values to load into Options +struct ConfigItem { + /// This is the list of parents + std::vector parents{}; + + /// This is the name + std::string name{}; + /// Listing of inputs + std::vector inputs{}; + + /// The list of parents and name joined by "." + CLI11_NODISCARD std::string fullname() const { + std::vector tmp = parents; + tmp.emplace_back(name); + return detail::join(tmp, "."); + } +}; + +/// This class provides a converter for configuration files. +class Config { + protected: + std::vector items{}; + + public: + /// Convert an app into a configuration + virtual std::string to_config(const App *, bool, bool, std::string) const = 0; + + /// Convert a configuration into an app + virtual std::vector from_config(std::istream &) const = 0; + + /// Get a flag value + CLI11_NODISCARD virtual std::string to_flag(const ConfigItem &item) const { + if(item.inputs.size() == 1) { + return item.inputs.at(0); + } + if(item.inputs.empty()) { + return "{}"; + } + throw ConversionError::TooManyInputsFlag(item.fullname()); // LCOV_EXCL_LINE + } + + /// Parse a config file, throw an error (ParseError:ConfigParseError or FileError) on failure + CLI11_NODISCARD std::vector from_file(const std::string &name) const { + std::ifstream input{name}; + if(!input.good()) + throw FileError::Missing(name); + + return from_config(input); + } + + /// Virtual destructor + virtual ~Config() = default; +}; + +/// This converter works with INI/TOML files; to write INI files use ConfigINI +class ConfigBase : public Config { + protected: + /// the character used for comments + char commentChar = '#'; + /// the character used to start an array '\0' is a default to not use + char arrayStart = '['; + /// the character used to end an array '\0' is a default to not use + char arrayEnd = ']'; + /// the character used to separate elements in an array + char arraySeparator = ','; + /// the character used separate the name from the value + char valueDelimiter = '='; + /// the character to use around strings + char stringQuote = '"'; + /// the character to use around single characters and literal strings + char literalQuote = '\''; + /// the maximum number of layers to allow + uint8_t maximumLayers{255}; + /// the separator used to separator parent layers + char parentSeparatorChar{'.'}; + /// Specify the configuration index to use for arrayed sections + int16_t configIndex{-1}; + /// Specify the configuration section that should be used + std::string configSection{}; + + public: + std::string + to_config(const App * /*app*/, bool default_also, bool write_description, std::string prefix) const override; + + std::vector from_config(std::istream &input) const override; + /// Specify the configuration for comment characters + ConfigBase *comment(char cchar) { + commentChar = cchar; + return this; + } + /// Specify the start and end characters for an array + ConfigBase *arrayBounds(char aStart, char aEnd) { + arrayStart = aStart; + arrayEnd = aEnd; + return this; + } + /// Specify the delimiter character for an array + ConfigBase *arrayDelimiter(char aSep) { + arraySeparator = aSep; + return this; + } + /// Specify the delimiter between a name and value + ConfigBase *valueSeparator(char vSep) { + valueDelimiter = vSep; + return this; + } + /// Specify the quote characters used around strings and literal strings + ConfigBase *quoteCharacter(char qString, char literalChar) { + stringQuote = qString; + literalQuote = literalChar; + return this; + } + /// Specify the maximum number of parents + ConfigBase *maxLayers(uint8_t layers) { + maximumLayers = layers; + return this; + } + /// Specify the separator to use for parent layers + ConfigBase *parentSeparator(char sep) { + parentSeparatorChar = sep; + return this; + } + /// get a reference to the configuration section + std::string §ionRef() { return configSection; } + /// get the section + CLI11_NODISCARD const std::string §ion() const { return configSection; } + /// specify a particular section of the configuration file to use + ConfigBase *section(const std::string §ionName) { + configSection = sectionName; + return this; + } + + /// get a reference to the configuration index + int16_t &indexRef() { return configIndex; } + /// get the section index + CLI11_NODISCARD int16_t index() const { return configIndex; } + /// specify a particular index in the section to use (-1) for all sections to use + ConfigBase *index(int16_t sectionIndex) { + configIndex = sectionIndex; + return this; + } +}; + +/// the default Config is the TOML file format +using ConfigTOML = ConfigBase; + +/// ConfigINI generates a "standard" INI compliant output +class ConfigINI : public ConfigTOML { + + public: + ConfigINI() { + commentChar = ';'; + arrayStart = '\0'; + arrayEnd = '\0'; + arraySeparator = ' '; + valueDelimiter = '='; + } +}; + + + +class Option; + +/// @defgroup validator_group Validators + +/// @brief Some validators that are provided +/// +/// These are simple `std::string(const std::string&)` validators that are useful. They return +/// a string if the validation fails. A custom struct is provided, as well, with the same user +/// semantics, but with the ability to provide a new type name. +/// @{ + +/// +class Validator { + protected: + /// This is the description function, if empty the description_ will be used + std::function desc_function_{[]() { return std::string{}; }}; + + /// This is the base function that is to be called. + /// Returns a string error message if validation fails. + std::function func_{[](std::string &) { return std::string{}; }}; + /// The name for search purposes of the Validator + std::string name_{}; + /// A Validator will only apply to an indexed value (-1 is all elements) + int application_index_ = -1; + /// Enable for Validator to allow it to be disabled if need be + bool active_{true}; + /// specify that a validator should not modify the input + bool non_modifying_{false}; + + Validator(std::string validator_desc, std::function func) + : desc_function_([validator_desc]() { return validator_desc; }), func_(std::move(func)) {} + + public: + Validator() = default; + /// Construct a Validator with just the description string + explicit Validator(std::string validator_desc) : desc_function_([validator_desc]() { return validator_desc; }) {} + /// Construct Validator from basic information + Validator(std::function op, std::string validator_desc, std::string validator_name = "") + : desc_function_([validator_desc]() { return validator_desc; }), func_(std::move(op)), + name_(std::move(validator_name)) {} + /// Set the Validator operation function + Validator &operation(std::function op) { + func_ = std::move(op); + return *this; + } + /// This is the required operator for a Validator - provided to help + /// users (CLI11 uses the member `func` directly) + std::string operator()(std::string &str) const; + + /// This is the required operator for a Validator - provided to help + /// users (CLI11 uses the member `func` directly) + std::string operator()(const std::string &str) const { + std::string value = str; + return (active_) ? func_(value) : std::string{}; + } + + /// Specify the type string + Validator &description(std::string validator_desc) { + desc_function_ = [validator_desc]() { return validator_desc; }; + return *this; + } + /// Specify the type string + CLI11_NODISCARD Validator description(std::string validator_desc) const; + + /// Generate type description information for the Validator + CLI11_NODISCARD std::string get_description() const { + if(active_) { + return desc_function_(); + } + return std::string{}; + } + /// Specify the type string + Validator &name(std::string validator_name) { + name_ = std::move(validator_name); + return *this; + } + /// Specify the type string + CLI11_NODISCARD Validator name(std::string validator_name) const { + Validator newval(*this); + newval.name_ = std::move(validator_name); + return newval; + } + /// Get the name of the Validator + CLI11_NODISCARD const std::string &get_name() const { return name_; } + /// Specify whether the Validator is active or not + Validator &active(bool active_val = true) { + active_ = active_val; + return *this; + } + /// Specify whether the Validator is active or not + CLI11_NODISCARD Validator active(bool active_val = true) const { + Validator newval(*this); + newval.active_ = active_val; + return newval; + } + + /// Specify whether the Validator can be modifying or not + Validator &non_modifying(bool no_modify = true) { + non_modifying_ = no_modify; + return *this; + } + /// Specify the application index of a validator + Validator &application_index(int app_index) { + application_index_ = app_index; + return *this; + } + /// Specify the application index of a validator + CLI11_NODISCARD Validator application_index(int app_index) const { + Validator newval(*this); + newval.application_index_ = app_index; + return newval; + } + /// Get the current value of the application index + CLI11_NODISCARD int get_application_index() const { return application_index_; } + /// Get a boolean if the validator is active + CLI11_NODISCARD bool get_active() const { return active_; } + + /// Get a boolean if the validator is allowed to modify the input returns true if it can modify the input + CLI11_NODISCARD bool get_modifying() const { return !non_modifying_; } + + /// Combining validators is a new validator. Type comes from left validator if function, otherwise only set if the + /// same. + Validator operator&(const Validator &other) const; + + /// Combining validators is a new validator. Type comes from left validator if function, otherwise only set if the + /// same. + Validator operator|(const Validator &other) const; + + /// Create a validator that fails when a given validator succeeds + Validator operator!() const; + + private: + void _merge_description(const Validator &val1, const Validator &val2, const std::string &merger); +}; + +/// Class wrapping some of the accessors of Validator +class CustomValidator : public Validator { + public: +}; +// The implementation of the built in validators is using the Validator class; +// the user is only expected to use the const (static) versions (since there's no setup). +// Therefore, this is in detail. +namespace detail { + +/// CLI enumeration of different file types +enum class path_type { nonexistent, file, directory }; + +/// get the type of the path from a file name +CLI11_INLINE path_type check_path(const char *file) noexcept; + +/// Check for an existing file (returns error message if check fails) +class ExistingFileValidator : public Validator { + public: + ExistingFileValidator(); +}; + +/// Check for an existing directory (returns error message if check fails) +class ExistingDirectoryValidator : public Validator { + public: + ExistingDirectoryValidator(); +}; + +/// Check for an existing path +class ExistingPathValidator : public Validator { + public: + ExistingPathValidator(); +}; + +/// Check for an non-existing path +class NonexistentPathValidator : public Validator { + public: + NonexistentPathValidator(); +}; + +/// Validate the given string is a legal ipv4 address +class IPV4Validator : public Validator { + public: + IPV4Validator(); +}; + +class EscapedStringTransformer : public Validator { + public: + EscapedStringTransformer(); +}; + +} // namespace detail + +// Static is not needed here, because global const implies static. + +/// Check for existing file (returns error message if check fails) +const detail::ExistingFileValidator ExistingFile; + +/// Check for an existing directory (returns error message if check fails) +const detail::ExistingDirectoryValidator ExistingDirectory; + +/// Check for an existing path +const detail::ExistingPathValidator ExistingPath; + +/// Check for an non-existing path +const detail::NonexistentPathValidator NonexistentPath; + +/// Check for an IP4 address +const detail::IPV4Validator ValidIPV4; + +/// convert escaped characters into their associated values +const detail::EscapedStringTransformer EscapedString; + +/// Validate the input as a particular type +template class TypeValidator : public Validator { + public: + explicit TypeValidator(const std::string &validator_name) + : Validator(validator_name, [](std::string &input_string) { + using CLI::detail::lexical_cast; + auto val = DesiredType(); + if(!lexical_cast(input_string, val)) { + return std::string("Failed parsing ") + input_string + " as a " + detail::type_name(); + } + return std::string(); + }) {} + TypeValidator() : TypeValidator(detail::type_name()) {} +}; + +/// Check for a number +const TypeValidator Number("NUMBER"); + +/// Modify a path if the file is a particular default location, can be used as Check or transform +/// with the error return optionally disabled +class FileOnDefaultPath : public Validator { + public: + explicit FileOnDefaultPath(std::string default_path, bool enableErrorReturn = true); +}; + +/// Produce a range (factory). Min and max are inclusive. +class Range : public Validator { + public: + /// This produces a range with min and max inclusive. + /// + /// Note that the constructor is templated, but the struct is not, so C++17 is not + /// needed to provide nice syntax for Range(a,b). + template + Range(T min_val, T max_val, const std::string &validator_name = std::string{}) : Validator(validator_name) { + if(validator_name.empty()) { + std::stringstream out; + out << detail::type_name() << " in [" << min_val << " - " << max_val << "]"; + description(out.str()); + } + + func_ = [min_val, max_val](std::string &input) { + using CLI::detail::lexical_cast; + T val; + bool converted = lexical_cast(input, val); + if((!converted) || (val < min_val || val > max_val)) { + std::stringstream out; + out << "Value " << input << " not in range ["; + out << min_val << " - " << max_val << "]"; + return out.str(); + } + return std::string{}; + }; + } + + /// Range of one value is 0 to value + template + explicit Range(T max_val, const std::string &validator_name = std::string{}) + : Range(static_cast(0), max_val, validator_name) {} +}; + +/// Check for a non negative number +const Range NonNegativeNumber((std::numeric_limits::max)(), "NONNEGATIVE"); + +/// Check for a positive valued number (val>0.0), ::min here is the smallest positive number +const Range PositiveNumber((std::numeric_limits::min)(), (std::numeric_limits::max)(), "POSITIVE"); + +/// Produce a bounded range (factory). Min and max are inclusive. +class Bound : public Validator { + public: + /// This bounds a value with min and max inclusive. + /// + /// Note that the constructor is templated, but the struct is not, so C++17 is not + /// needed to provide nice syntax for Range(a,b). + template Bound(T min_val, T max_val) { + std::stringstream out; + out << detail::type_name() << " bounded to [" << min_val << " - " << max_val << "]"; + description(out.str()); + + func_ = [min_val, max_val](std::string &input) { + using CLI::detail::lexical_cast; + T val; + bool converted = lexical_cast(input, val); + if(!converted) { + return std::string("Value ") + input + " could not be converted"; + } + if(val < min_val) + input = detail::to_string(min_val); + else if(val > max_val) + input = detail::to_string(max_val); + + return std::string{}; + }; + } + + /// Range of one value is 0 to value + template explicit Bound(T max_val) : Bound(static_cast(0), max_val) {} +}; + +namespace detail { +template ::type>::value, detail::enabler> = detail::dummy> +auto smart_deref(T value) -> decltype(*value) { + return *value; +} + +template < + typename T, + enable_if_t::type>::value, detail::enabler> = detail::dummy> +typename std::remove_reference::type &smart_deref(T &value) { + return value; +} +/// Generate a string representation of a set +template std::string generate_set(const T &set) { + using element_t = typename detail::element_type::type; + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + std::string out(1, '{'); + out.append(detail::join( + detail::smart_deref(set), + [](const iteration_type_t &v) { return detail::pair_adaptor::first(v); }, + ",")); + out.push_back('}'); + return out; +} + +/// Generate a string representation of a map +template std::string generate_map(const T &map, bool key_only = false) { + using element_t = typename detail::element_type::type; + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + std::string out(1, '{'); + out.append(detail::join( + detail::smart_deref(map), + [key_only](const iteration_type_t &v) { + std::string res{detail::to_string(detail::pair_adaptor::first(v))}; + + if(!key_only) { + res.append("->"); + res += detail::to_string(detail::pair_adaptor::second(v)); + } + return res; + }, + ",")); + out.push_back('}'); + return out; +} + +template struct has_find { + template + static auto test(int) -> decltype(std::declval().find(std::declval()), std::true_type()); + template static auto test(...) -> decltype(std::false_type()); + + static const auto value = decltype(test(0))::value; + using type = std::integral_constant; +}; + +/// A search function +template ::value, detail::enabler> = detail::dummy> +auto search(const T &set, const V &val) -> std::pair { + using element_t = typename detail::element_type::type; + auto &setref = detail::smart_deref(set); + auto it = std::find_if(std::begin(setref), std::end(setref), [&val](decltype(*std::begin(setref)) v) { + return (detail::pair_adaptor::first(v) == val); + }); + return {(it != std::end(setref)), it}; +} + +/// A search function that uses the built in find function +template ::value, detail::enabler> = detail::dummy> +auto search(const T &set, const V &val) -> std::pair { + auto &setref = detail::smart_deref(set); + auto it = setref.find(val); + return {(it != std::end(setref)), it}; +} + +/// A search function with a filter function +template +auto search(const T &set, const V &val, const std::function &filter_function) + -> std::pair { + using element_t = typename detail::element_type::type; + // do the potentially faster first search + auto res = search(set, val); + if((res.first) || (!(filter_function))) { + return res; + } + // if we haven't found it do the longer linear search with all the element translations + auto &setref = detail::smart_deref(set); + auto it = std::find_if(std::begin(setref), std::end(setref), [&](decltype(*std::begin(setref)) v) { + V a{detail::pair_adaptor::first(v)}; + a = filter_function(a); + return (a == val); + }); + return {(it != std::end(setref)), it}; +} + +// the following suggestion was made by Nikita Ofitserov(@himikof) +// done in templates to prevent compiler warnings on negation of unsigned numbers + +/// Do a check for overflow on signed numbers +template +inline typename std::enable_if::value, T>::type overflowCheck(const T &a, const T &b) { + if((a > 0) == (b > 0)) { + return ((std::numeric_limits::max)() / (std::abs)(a) < (std::abs)(b)); + } + return ((std::numeric_limits::min)() / (std::abs)(a) > -(std::abs)(b)); +} +/// Do a check for overflow on unsigned numbers +template +inline typename std::enable_if::value, T>::type overflowCheck(const T &a, const T &b) { + return ((std::numeric_limits::max)() / a < b); +} + +/// Performs a *= b; if it doesn't cause integer overflow. Returns false otherwise. +template typename std::enable_if::value, bool>::type checked_multiply(T &a, T b) { + if(a == 0 || b == 0 || a == 1 || b == 1) { + a *= b; + return true; + } + if(a == (std::numeric_limits::min)() || b == (std::numeric_limits::min)()) { + return false; + } + if(overflowCheck(a, b)) { + return false; + } + a *= b; + return true; +} + +/// Performs a *= b; if it doesn't equal infinity. Returns false otherwise. +template +typename std::enable_if::value, bool>::type checked_multiply(T &a, T b) { + T c = a * b; + if(std::isinf(c) && !std::isinf(a) && !std::isinf(b)) { + return false; + } + a = c; + return true; +} + +} // namespace detail +/// Verify items are in a set +class IsMember : public Validator { + public: + using filter_fn_t = std::function; + + /// This allows in-place construction using an initializer list + template + IsMember(std::initializer_list values, Args &&...args) + : IsMember(std::vector(values), std::forward(args)...) {} + + /// This checks to see if an item is in a set (empty function) + template explicit IsMember(T &&set) : IsMember(std::forward(set), nullptr) {} + + /// This checks to see if an item is in a set: pointer or copy version. You can pass in a function that will filter + /// both sides of the comparison before computing the comparison. + template explicit IsMember(T set, F filter_function) { + + // Get the type of the contained item - requires a container have ::value_type + // if the type does not have first_type and second_type, these are both value_type + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + + using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones + // (const char * to std::string) + + // Make a local copy of the filter function, using a std::function if not one already + std::function filter_fn = filter_function; + + // This is the type name for help, it will take the current version of the set contents + desc_function_ = [set]() { return detail::generate_set(detail::smart_deref(set)); }; + + // This is the function that validates + // It stores a copy of the set pointer-like, so shared_ptr will stay alive + func_ = [set, filter_fn](std::string &input) { + using CLI::detail::lexical_cast; + local_item_t b; + if(!lexical_cast(input, b)) { + throw ValidationError(input); // name is added later + } + if(filter_fn) { + b = filter_fn(b); + } + auto res = detail::search(set, b, filter_fn); + if(res.first) { + // Make sure the version in the input string is identical to the one in the set + if(filter_fn) { + input = detail::value_string(detail::pair_adaptor::first(*(res.second))); + } + + // Return empty error string (success) + return std::string{}; + } + + // If you reach this point, the result was not found + return input + " not in " + detail::generate_set(detail::smart_deref(set)); + }; + } + + /// You can pass in as many filter functions as you like, they nest (string only currently) + template + IsMember(T &&set, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&...other) + : IsMember( + std::forward(set), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} +}; + +/// definition of the default transformation object +template using TransformPairs = std::vector>; + +/// Translate named items to other or a value set +class Transformer : public Validator { + public: + using filter_fn_t = std::function; + + /// This allows in-place construction + template + Transformer(std::initializer_list> values, Args &&...args) + : Transformer(TransformPairs(values), std::forward(args)...) {} + + /// direct map of std::string to std::string + template explicit Transformer(T &&mapping) : Transformer(std::forward(mapping), nullptr) {} + + /// This checks to see if an item is in a set: pointer or copy version. You can pass in a function that will filter + /// both sides of the comparison before computing the comparison. + template explicit Transformer(T mapping, F filter_function) { + + static_assert(detail::pair_adaptor::type>::value, + "mapping must produce value pairs"); + // Get the type of the contained item - requires a container have ::value_type + // if the type does not have first_type and second_type, these are both value_type + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using local_item_t = typename IsMemberType::type; // Will convert bad types to good ones + // (const char * to std::string) + + // Make a local copy of the filter function, using a std::function if not one already + std::function filter_fn = filter_function; + + // This is the type name for help, it will take the current version of the set contents + desc_function_ = [mapping]() { return detail::generate_map(detail::smart_deref(mapping)); }; + + func_ = [mapping, filter_fn](std::string &input) { + using CLI::detail::lexical_cast; + local_item_t b; + if(!lexical_cast(input, b)) { + return std::string(); + // there is no possible way we can match anything in the mapping if we can't convert so just return + } + if(filter_fn) { + b = filter_fn(b); + } + auto res = detail::search(mapping, b, filter_fn); + if(res.first) { + input = detail::value_string(detail::pair_adaptor::second(*res.second)); + } + return std::string{}; + }; + } + + /// You can pass in as many filter functions as you like, they nest + template + Transformer(T &&mapping, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&...other) + : Transformer( + std::forward(mapping), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} +}; + +/// translate named items to other or a value set +class CheckedTransformer : public Validator { + public: + using filter_fn_t = std::function; + + /// This allows in-place construction + template + CheckedTransformer(std::initializer_list> values, Args &&...args) + : CheckedTransformer(TransformPairs(values), std::forward(args)...) {} + + /// direct map of std::string to std::string + template explicit CheckedTransformer(T mapping) : CheckedTransformer(std::move(mapping), nullptr) {} + + /// This checks to see if an item is in a set: pointer or copy version. You can pass in a function that will filter + /// both sides of the comparison before computing the comparison. + template explicit CheckedTransformer(T mapping, F filter_function) { + + static_assert(detail::pair_adaptor::type>::value, + "mapping must produce value pairs"); + // Get the type of the contained item - requires a container have ::value_type + // if the type does not have first_type and second_type, these are both value_type + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using local_item_t = typename IsMemberType::type; // Will convert bad types to good ones + // (const char * to std::string) + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + + // Make a local copy of the filter function, using a std::function if not one already + std::function filter_fn = filter_function; + + auto tfunc = [mapping]() { + std::string out("value in "); + out += detail::generate_map(detail::smart_deref(mapping)) + " OR {"; + out += detail::join( + detail::smart_deref(mapping), + [](const iteration_type_t &v) { return detail::to_string(detail::pair_adaptor::second(v)); }, + ","); + out.push_back('}'); + return out; + }; + + desc_function_ = tfunc; + + func_ = [mapping, tfunc, filter_fn](std::string &input) { + using CLI::detail::lexical_cast; + local_item_t b; + bool converted = lexical_cast(input, b); + if(converted) { + if(filter_fn) { + b = filter_fn(b); + } + auto res = detail::search(mapping, b, filter_fn); + if(res.first) { + input = detail::value_string(detail::pair_adaptor::second(*res.second)); + return std::string{}; + } + } + for(const auto &v : detail::smart_deref(mapping)) { + auto output_string = detail::value_string(detail::pair_adaptor::second(v)); + if(output_string == input) { + return std::string(); + } + } + + return "Check " + input + " " + tfunc() + " FAILED"; + }; + } + + /// You can pass in as many filter functions as you like, they nest + template + CheckedTransformer(T &&mapping, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&...other) + : CheckedTransformer( + std::forward(mapping), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} +}; + +/// Helper function to allow ignore_case to be passed to IsMember or Transform +inline std::string ignore_case(std::string item) { return detail::to_lower(item); } + +/// Helper function to allow ignore_underscore to be passed to IsMember or Transform +inline std::string ignore_underscore(std::string item) { return detail::remove_underscore(item); } + +/// Helper function to allow checks to ignore spaces to be passed to IsMember or Transform +inline std::string ignore_space(std::string item) { + item.erase(std::remove(std::begin(item), std::end(item), ' '), std::end(item)); + item.erase(std::remove(std::begin(item), std::end(item), '\t'), std::end(item)); + return item; +} + +/// Multiply a number by a factor using given mapping. +/// Can be used to write transforms for SIZE or DURATION inputs. +/// +/// Example: +/// With mapping = `{"b"->1, "kb"->1024, "mb"->1024*1024}` +/// one can recognize inputs like "100", "12kb", "100 MB", +/// that will be automatically transformed to 100, 14448, 104857600. +/// +/// Output number type matches the type in the provided mapping. +/// Therefore, if it is required to interpret real inputs like "0.42 s", +/// the mapping should be of a type or . +class AsNumberWithUnit : public Validator { + public: + /// Adjust AsNumberWithUnit behavior. + /// CASE_SENSITIVE/CASE_INSENSITIVE controls how units are matched. + /// UNIT_OPTIONAL/UNIT_REQUIRED throws ValidationError + /// if UNIT_REQUIRED is set and unit literal is not found. + enum Options { + CASE_SENSITIVE = 0, + CASE_INSENSITIVE = 1, + UNIT_OPTIONAL = 0, + UNIT_REQUIRED = 2, + DEFAULT = CASE_INSENSITIVE | UNIT_OPTIONAL + }; + + template + explicit AsNumberWithUnit(std::map mapping, + Options opts = DEFAULT, + const std::string &unit_name = "UNIT") { + description(generate_description(unit_name, opts)); + validate_mapping(mapping, opts); + + // transform function + func_ = [mapping, opts](std::string &input) -> std::string { + Number num{}; + + detail::rtrim(input); + if(input.empty()) { + throw ValidationError("Input is empty"); + } + + // Find split position between number and prefix + auto unit_begin = input.end(); + while(unit_begin > input.begin() && std::isalpha(*(unit_begin - 1), std::locale())) { + --unit_begin; + } + + std::string unit{unit_begin, input.end()}; + input.resize(static_cast(std::distance(input.begin(), unit_begin))); + detail::trim(input); + + if(opts & UNIT_REQUIRED && unit.empty()) { + throw ValidationError("Missing mandatory unit"); + } + if(opts & CASE_INSENSITIVE) { + unit = detail::to_lower(unit); + } + if(unit.empty()) { + using CLI::detail::lexical_cast; + if(!lexical_cast(input, num)) { + throw ValidationError(std::string("Value ") + input + " could not be converted to " + + detail::type_name()); + } + // No need to modify input if no unit passed + return {}; + } + + // find corresponding factor + auto it = mapping.find(unit); + if(it == mapping.end()) { + throw ValidationError(unit + + " unit not recognized. " + "Allowed values: " + + detail::generate_map(mapping, true)); + } + + if(!input.empty()) { + using CLI::detail::lexical_cast; + bool converted = lexical_cast(input, num); + if(!converted) { + throw ValidationError(std::string("Value ") + input + " could not be converted to " + + detail::type_name()); + } + // perform safe multiplication + bool ok = detail::checked_multiply(num, it->second); + if(!ok) { + throw ValidationError(detail::to_string(num) + " multiplied by " + unit + + " factor would cause number overflow. Use smaller value."); + } + } else { + num = static_cast(it->second); + } + + input = detail::to_string(num); + + return {}; + }; + } + + private: + /// Check that mapping contains valid units. + /// Update mapping for CASE_INSENSITIVE mode. + template static void validate_mapping(std::map &mapping, Options opts) { + for(auto &kv : mapping) { + if(kv.first.empty()) { + throw ValidationError("Unit must not be empty."); + } + if(!detail::isalpha(kv.first)) { + throw ValidationError("Unit must contain only letters."); + } + } + + // make all units lowercase if CASE_INSENSITIVE + if(opts & CASE_INSENSITIVE) { + std::map lower_mapping; + for(auto &kv : mapping) { + auto s = detail::to_lower(kv.first); + if(lower_mapping.count(s)) { + throw ValidationError(std::string("Several matching lowercase unit representations are found: ") + + s); + } + lower_mapping[detail::to_lower(kv.first)] = kv.second; + } + mapping = std::move(lower_mapping); + } + } + + /// Generate description like this: NUMBER [UNIT] + template static std::string generate_description(const std::string &name, Options opts) { + std::stringstream out; + out << detail::type_name() << ' '; + if(opts & UNIT_REQUIRED) { + out << name; + } else { + out << '[' << name << ']'; + } + return out.str(); + } +}; + +inline AsNumberWithUnit::Options operator|(const AsNumberWithUnit::Options &a, const AsNumberWithUnit::Options &b) { + return static_cast(static_cast(a) | static_cast(b)); +} + +/// Converts a human-readable size string (with unit literal) to uin64_t size. +/// Example: +/// "100" => 100 +/// "1 b" => 100 +/// "10Kb" => 10240 // you can configure this to be interpreted as kilobyte (*1000) or kibibyte (*1024) +/// "10 KB" => 10240 +/// "10 kb" => 10240 +/// "10 kib" => 10240 // *i, *ib are always interpreted as *bibyte (*1024) +/// "10kb" => 10240 +/// "2 MB" => 2097152 +/// "2 EiB" => 2^61 // Units up to exibyte are supported +class AsSizeValue : public AsNumberWithUnit { + public: + using result_t = std::uint64_t; + + /// If kb_is_1000 is true, + /// interpret 'kb', 'k' as 1000 and 'kib', 'ki' as 1024 + /// (same applies to higher order units as well). + /// Otherwise, interpret all literals as factors of 1024. + /// The first option is formally correct, but + /// the second interpretation is more wide-spread + /// (see https://en.wikipedia.org/wiki/Binary_prefix). + explicit AsSizeValue(bool kb_is_1000); + + private: + /// Get mapping + static std::map init_mapping(bool kb_is_1000); + + /// Cache calculated mapping + static std::map get_mapping(bool kb_is_1000); +}; + +namespace detail { +/// Split a string into a program name and command line arguments +/// the string is assumed to contain a file name followed by other arguments +/// the return value contains is a pair with the first argument containing the program name and the second +/// everything else. +CLI11_INLINE std::pair split_program_name(std::string commandline); + +} // namespace detail +/// @} + + + + +CLI11_INLINE std::string Validator::operator()(std::string &str) const { + std::string retstring; + if(active_) { + if(non_modifying_) { + std::string value = str; + retstring = func_(value); + } else { + retstring = func_(str); + } + } + return retstring; +} + +CLI11_NODISCARD CLI11_INLINE Validator Validator::description(std::string validator_desc) const { + Validator newval(*this); + newval.desc_function_ = [validator_desc]() { return validator_desc; }; + return newval; +} + +CLI11_INLINE Validator Validator::operator&(const Validator &other) const { + Validator newval; + + newval._merge_description(*this, other, " AND "); + + // Give references (will make a copy in lambda function) + const std::function &f1 = func_; + const std::function &f2 = other.func_; + + newval.func_ = [f1, f2](std::string &input) { + std::string s1 = f1(input); + std::string s2 = f2(input); + if(!s1.empty() && !s2.empty()) + return std::string("(") + s1 + ") AND (" + s2 + ")"; + return s1 + s2; + }; + + newval.active_ = active_ && other.active_; + newval.application_index_ = application_index_; + return newval; +} + +CLI11_INLINE Validator Validator::operator|(const Validator &other) const { + Validator newval; + + newval._merge_description(*this, other, " OR "); + + // Give references (will make a copy in lambda function) + const std::function &f1 = func_; + const std::function &f2 = other.func_; + + newval.func_ = [f1, f2](std::string &input) { + std::string s1 = f1(input); + std::string s2 = f2(input); + if(s1.empty() || s2.empty()) + return std::string(); + + return std::string("(") + s1 + ") OR (" + s2 + ")"; + }; + newval.active_ = active_ && other.active_; + newval.application_index_ = application_index_; + return newval; +} + +CLI11_INLINE Validator Validator::operator!() const { + Validator newval; + const std::function &dfunc1 = desc_function_; + newval.desc_function_ = [dfunc1]() { + auto str = dfunc1(); + return (!str.empty()) ? std::string("NOT ") + str : std::string{}; + }; + // Give references (will make a copy in lambda function) + const std::function &f1 = func_; + + newval.func_ = [f1, dfunc1](std::string &test) -> std::string { + std::string s1 = f1(test); + if(s1.empty()) { + return std::string("check ") + dfunc1() + " succeeded improperly"; + } + return std::string{}; + }; + newval.active_ = active_; + newval.application_index_ = application_index_; + return newval; +} + +CLI11_INLINE void +Validator::_merge_description(const Validator &val1, const Validator &val2, const std::string &merger) { + + const std::function &dfunc1 = val1.desc_function_; + const std::function &dfunc2 = val2.desc_function_; + + desc_function_ = [=]() { + std::string f1 = dfunc1(); + std::string f2 = dfunc2(); + if((f1.empty()) || (f2.empty())) { + return f1 + f2; + } + return std::string(1, '(') + f1 + ')' + merger + '(' + f2 + ')'; + }; +} + +namespace detail { + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +CLI11_INLINE path_type check_path(const char *file) noexcept { + std::error_code ec; + auto stat = std::filesystem::status(to_path(file), ec); + if(ec) { + return path_type::nonexistent; + } + switch(stat.type()) { + case std::filesystem::file_type::none: // LCOV_EXCL_LINE + case std::filesystem::file_type::not_found: + return path_type::nonexistent; // LCOV_EXCL_LINE + case std::filesystem::file_type::directory: + return path_type::directory; + case std::filesystem::file_type::symlink: + case std::filesystem::file_type::block: + case std::filesystem::file_type::character: + case std::filesystem::file_type::fifo: + case std::filesystem::file_type::socket: + case std::filesystem::file_type::regular: + case std::filesystem::file_type::unknown: + default: + return path_type::file; + } +} +#else +CLI11_INLINE path_type check_path(const char *file) noexcept { +#if defined(_MSC_VER) + struct __stat64 buffer; + if(_stat64(file, &buffer) == 0) { + return ((buffer.st_mode & S_IFDIR) != 0) ? path_type::directory : path_type::file; + } +#else + struct stat buffer; + if(stat(file, &buffer) == 0) { + return ((buffer.st_mode & S_IFDIR) != 0) ? path_type::directory : path_type::file; + } +#endif + return path_type::nonexistent; +} +#endif + +CLI11_INLINE ExistingFileValidator::ExistingFileValidator() : Validator("FILE") { + func_ = [](std::string &filename) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { + return "File does not exist: " + filename; + } + if(path_result == path_type::directory) { + return "File is actually a directory: " + filename; + } + return std::string(); + }; +} + +CLI11_INLINE ExistingDirectoryValidator::ExistingDirectoryValidator() : Validator("DIR") { + func_ = [](std::string &filename) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { + return "Directory does not exist: " + filename; + } + if(path_result == path_type::file) { + return "Directory is actually a file: " + filename; + } + return std::string(); + }; +} + +CLI11_INLINE ExistingPathValidator::ExistingPathValidator() : Validator("PATH(existing)") { + func_ = [](std::string &filename) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { + return "Path does not exist: " + filename; + } + return std::string(); + }; +} + +CLI11_INLINE NonexistentPathValidator::NonexistentPathValidator() : Validator("PATH(non-existing)") { + func_ = [](std::string &filename) { + auto path_result = check_path(filename.c_str()); + if(path_result != path_type::nonexistent) { + return "Path already exists: " + filename; + } + return std::string(); + }; +} + +CLI11_INLINE IPV4Validator::IPV4Validator() : Validator("IPV4") { + func_ = [](std::string &ip_addr) { + auto result = CLI::detail::split(ip_addr, '.'); + if(result.size() != 4) { + return std::string("Invalid IPV4 address must have four parts (") + ip_addr + ')'; + } + int num = 0; + for(const auto &var : result) { + using CLI::detail::lexical_cast; + bool retval = lexical_cast(var, num); + if(!retval) { + return std::string("Failed parsing number (") + var + ')'; + } + if(num < 0 || num > 255) { + return std::string("Each IP number must be between 0 and 255 ") + var; + } + } + return std::string{}; + }; +} + +CLI11_INLINE EscapedStringTransformer::EscapedStringTransformer() { + func_ = [](std::string &str) { + try { + if(str.size() > 1 && (str.front() == '\"' || str.front() == '\'' || str.front() == '`') && + str.front() == str.back()) { + process_quoted_string(str); + } else if(str.find_first_of('\\') != std::string::npos) { + if(detail::is_binary_escaped_string(str)) { + str = detail::extract_binary_string(str); + } else { + str = remove_escaped_characters(str); + } + } + return std::string{}; + } catch(const std::invalid_argument &ia) { + return std::string(ia.what()); + } + }; +} +} // namespace detail + +CLI11_INLINE FileOnDefaultPath::FileOnDefaultPath(std::string default_path, bool enableErrorReturn) + : Validator("FILE") { + func_ = [default_path, enableErrorReturn](std::string &filename) { + auto path_result = detail::check_path(filename.c_str()); + if(path_result == detail::path_type::nonexistent) { + std::string test_file_path = default_path; + if(default_path.back() != '/' && default_path.back() != '\\') { + // Add folder separator + test_file_path += '/'; + } + test_file_path.append(filename); + path_result = detail::check_path(test_file_path.c_str()); + if(path_result == detail::path_type::file) { + filename = test_file_path; + } else { + if(enableErrorReturn) { + return "File does not exist: " + filename; + } + } + } + return std::string{}; + }; +} + +CLI11_INLINE AsSizeValue::AsSizeValue(bool kb_is_1000) : AsNumberWithUnit(get_mapping(kb_is_1000)) { + if(kb_is_1000) { + description("SIZE [b, kb(=1000b), kib(=1024b), ...]"); + } else { + description("SIZE [b, kb(=1024b), ...]"); + } +} + +CLI11_INLINE std::map AsSizeValue::init_mapping(bool kb_is_1000) { + std::map m; + result_t k_factor = kb_is_1000 ? 1000 : 1024; + result_t ki_factor = 1024; + result_t k = 1; + result_t ki = 1; + m["b"] = 1; + for(std::string p : {"k", "m", "g", "t", "p", "e"}) { + k *= k_factor; + ki *= ki_factor; + m[p] = k; + m[p + "b"] = k; + m[p + "i"] = ki; + m[p + "ib"] = ki; + } + return m; +} + +CLI11_INLINE std::map AsSizeValue::get_mapping(bool kb_is_1000) { + if(kb_is_1000) { + static auto m = init_mapping(true); + return m; + } + static auto m = init_mapping(false); + return m; +} + +namespace detail { + +CLI11_INLINE std::pair split_program_name(std::string commandline) { + // try to determine the programName + std::pair vals; + trim(commandline); + auto esp = commandline.find_first_of(' ', 1); + while(detail::check_path(commandline.substr(0, esp).c_str()) != path_type::file) { + esp = commandline.find_first_of(' ', esp + 1); + if(esp == std::string::npos) { + // if we have reached the end and haven't found a valid file just assume the first argument is the + // program name + if(commandline[0] == '"' || commandline[0] == '\'' || commandline[0] == '`') { + bool embeddedQuote = false; + auto keyChar = commandline[0]; + auto end = commandline.find_first_of(keyChar, 1); + while((end != std::string::npos) && (commandline[end - 1] == '\\')) { // deal with escaped quotes + end = commandline.find_first_of(keyChar, end + 1); + embeddedQuote = true; + } + if(end != std::string::npos) { + vals.first = commandline.substr(1, end - 1); + esp = end + 1; + if(embeddedQuote) { + vals.first = find_and_replace(vals.first, std::string("\\") + keyChar, std::string(1, keyChar)); + } + } else { + esp = commandline.find_first_of(' ', 1); + } + } else { + esp = commandline.find_first_of(' ', 1); + } + + break; + } + } + if(vals.first.empty()) { + vals.first = commandline.substr(0, esp); + rtrim(vals.first); + } + + // strip the program name + vals.second = (esp < commandline.length() - 1) ? commandline.substr(esp + 1) : std::string{}; + ltrim(vals.second); + return vals; +} + +} // namespace detail +/// @} + + + + +class Option; +class App; + +/// This enum signifies the type of help requested +/// +/// This is passed in by App; all user classes must accept this as +/// the second argument. + +enum class AppFormatMode { + Normal, ///< The normal, detailed help + All, ///< A fully expanded help + Sub, ///< Used when printed as part of expanded subcommand +}; + +/// This is the minimum requirements to run a formatter. +/// +/// A user can subclass this is if they do not care at all +/// about the structure in CLI::Formatter. +class FormatterBase { + protected: + /// @name Options + ///@{ + + /// The width of the first column + std::size_t column_width_{30}; + + /// @brief The required help printout labels (user changeable) + /// Values are Needs, Excludes, etc. + std::map labels_{}; + + ///@} + /// @name Basic + ///@{ + + public: + FormatterBase() = default; + FormatterBase(const FormatterBase &) = default; + FormatterBase(FormatterBase &&) = default; + FormatterBase &operator=(const FormatterBase &) = default; + FormatterBase &operator=(FormatterBase &&) = default; + + /// Adding a destructor in this form to work around bug in GCC 4.7 + virtual ~FormatterBase() noexcept {} // NOLINT(modernize-use-equals-default) + + /// This is the key method that puts together help + virtual std::string make_help(const App *, std::string, AppFormatMode) const = 0; + + ///@} + /// @name Setters + ///@{ + + /// Set the "REQUIRED" label + void label(std::string key, std::string val) { labels_[key] = val; } + + /// Set the column width + void column_width(std::size_t val) { column_width_ = val; } + + ///@} + /// @name Getters + ///@{ + + /// Get the current value of a name (REQUIRED, etc.) + CLI11_NODISCARD std::string get_label(std::string key) const { + if(labels_.find(key) == labels_.end()) + return key; + return labels_.at(key); + } + + /// Get the current column width + CLI11_NODISCARD std::size_t get_column_width() const { return column_width_; } + + ///@} +}; + +/// This is a specialty override for lambda functions +class FormatterLambda final : public FormatterBase { + using funct_t = std::function; + + /// The lambda to hold and run + funct_t lambda_; + + public: + /// Create a FormatterLambda with a lambda function + explicit FormatterLambda(funct_t funct) : lambda_(std::move(funct)) {} + + /// Adding a destructor (mostly to make GCC 4.7 happy) + ~FormatterLambda() noexcept override {} // NOLINT(modernize-use-equals-default) + + /// This will simply call the lambda function + std::string make_help(const App *app, std::string name, AppFormatMode mode) const override { + return lambda_(app, name, mode); + } +}; + +/// This is the default Formatter for CLI11. It pretty prints help output, and is broken into quite a few +/// overridable methods, to be highly customizable with minimal effort. +class Formatter : public FormatterBase { + public: + Formatter() = default; + Formatter(const Formatter &) = default; + Formatter(Formatter &&) = default; + Formatter &operator=(const Formatter &) = default; + Formatter &operator=(Formatter &&) = default; + + /// @name Overridables + ///@{ + + /// This prints out a group of options with title + /// + CLI11_NODISCARD virtual std::string + make_group(std::string group, bool is_positional, std::vector opts) const; + + /// This prints out just the positionals "group" + virtual std::string make_positionals(const App *app) const; + + /// This prints out all the groups of options + std::string make_groups(const App *app, AppFormatMode mode) const; + + /// This prints out all the subcommands + virtual std::string make_subcommands(const App *app, AppFormatMode mode) const; + + /// This prints out a subcommand + virtual std::string make_subcommand(const App *sub) const; + + /// This prints out a subcommand in help-all + virtual std::string make_expanded(const App *sub) const; + + /// This prints out all the groups of options + virtual std::string make_footer(const App *app) const; + + /// This displays the description line + virtual std::string make_description(const App *app) const; + + /// This displays the usage line + virtual std::string make_usage(const App *app, std::string name) const; + + /// This puts everything together + std::string make_help(const App * /*app*/, std::string, AppFormatMode) const override; + + ///@} + /// @name Options + ///@{ + + /// This prints out an option help line, either positional or optional form + virtual std::string make_option(const Option *opt, bool is_positional) const { + std::stringstream out; + detail::format_help( + out, make_option_name(opt, is_positional) + make_option_opts(opt), make_option_desc(opt), column_width_); + return out.str(); + } + + /// @brief This is the name part of an option, Default: left column + virtual std::string make_option_name(const Option *, bool) const; + + /// @brief This is the options part of the name, Default: combined into left column + virtual std::string make_option_opts(const Option *) const; + + /// @brief This is the description. Default: Right column, on new line if left column too large + virtual std::string make_option_desc(const Option *) const; + + /// @brief This is used to print the name on the USAGE line + virtual std::string make_option_usage(const Option *opt) const; + + ///@} +}; + + + + +using results_t = std::vector; +/// callback function definition +using callback_t = std::function; + +class Option; +class App; + +using Option_p = std::unique_ptr